How to Set Up ElevenLabs Voice Cloning for Personalized Customer Interactions

flowchart LR
    A[Customer Call] --> B[VAPI Assistant]
    B --> C[Deepgram STT]
    C --> D[GPT-4 Response]
    D --> E[ElevenLabs Voice Clone]
    E --> F[Synthesized Audio]
    F --> G[Phone Line]
    G --> A
    B --> H[Webhook Server]
    H --> I[Call Analytics]

graph LR
    A[Microphone] --> B[Audio Buffer]
    B --> C[Voice Activity Detection]
    C -->|Speech Detected| D[Speech-to-Text]
    C -->|Silence| E[Error: No Speech Detected]
    D --> F[Intent Detection]
    F --> G[Response Generation]
    G --> H[Text-to-Speech]
    H --> I[Speaker]
    D -->|Error: Unrecognized Speech| J[Error Handling]
    J --> F
    F -->|Error: No Intent| K[Fallback Response]
    K --> G

npm install express dotenv node-fetch

VAPI_API_KEY=your_vapi_key
VAPI_PRIVATE_KEY=your_private_key
ELEVENLABS_API_KEY=your_elevenlabs_key
ELEVENLABS_VOICE_ID=your_cloned_voice_id
WEBHOOK_URL=https://your-domain.com/webhook/vapi
WEBHOOK_SECRET=your_webhook_secret

// createAssistant.js - Production assistant creation
require('dotenv').config();
const fetch = require('node-fetch');

async function createVoiceCloneAssistant() {
  try {
    const response = await fetch('https://api.vapi.ai/assistant', {
      method: 'POST',
      headers: {
        'Authorization': 'Bearer ' + process.env.VAPI_API_KEY,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        name: "Customer Support Clone",
        model: {
          provider: "openai",
          model: "gpt-4",
          temperature: 0.7,
          maxTokens: 150,
          messages: [{
            role: "system",
            content: "You are Sarah, a friendly customer support agent. Keep responses under 50 words for natural conversation flow."
          }]
        },
        voice: {
          provider: "11labs",
          voiceId: process.env.ELEVENLABS_VOICE_ID,
          model: "eleven_turbo_v2", // Lowest latency for real-time
          stability: 0.5, // Lower = more expressive, higher = more consistent
          similarityBoost: 0.75, // How closely to match the original voice
          optimizeStreamingLatency: 3, // 0-4 scale, 3 = balanced
          enableSsmlParsing: true
        },
        transcriber: {
          provider: "deepgram",
          model: "nova-2",
          language: "en-US",
          smartFormat: true
        },
        firstMessage: "Hi, this is Sarah from customer support. How can I help you today?",
        serverUrl: process.env.WEBHOOK_URL,
        serverUrlSecret: process.env.WEBHOOK_SECRET,
        endCallMessage: "Thanks for calling. Have a great day!",
        endCallPhrases: ["goodbye", "that's all", "thank you bye"]
      })
    });

    if (!response.ok) {
      const error = await response.json();
      if (error.message?.includes('quota')) {
        console.error('ElevenLabs quota exceeded - implement fallback voice');
        throw new Error('Voice cloning quota exhausted');
      }
      throw new Error(`HTTP error! status: ${response.status}`);
    }

    const assistant = await response.json();
    console.log('Assistant created:', assistant.id);
    return assistant;
  } catch (error) {
    console.error('Failed to create assistant:', error);
    throw error;
  }
}

createVoiceCloneAssistant();

// server.js - Production webhook handler
const express = require('express');
const crypto = require('crypto');
const app = express();

app.use(express.json());

// Webhook signature validation - REQUIRED for production
function validateWebhook(req) {
  const signature = req.headers['x-vapi-signature'];
  if (!signature) return false;
  
  const payload = JSON.stringify(req.body);
  const hash = crypto
    .createHmac('sha256', process.env.WEBHOOK_SECRET)
    .update(payload)
    .digest('hex');
  
  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(hash)
  );
}

// Session state tracking - prevents race conditions
const activeSessions = new Map();
const SESSION_TTL = 3600000; // 1 hour

app.post('/webhook/vapi', async (req, res) => {
  if (!validateWebhook(req)) {
    console.error('Invalid webhook signature');
    return res.status(401).json({ error: 'Invalid signature' });
  }

  const { message } = req.body;
  const callId = message.call?.id;

  // Track session state
  if (message.type === 'call-start') {
    activeSessions.set(callId, {
      startTime: Date.now(),
      voiceErrors: 0,
      latencyWarnings: 0,
      isSpeaking: false,
      audioBuffer: []
    });
    setTimeout(() => activeSessions.delete(callId), SESSION_TTL);
  }

  // Handle voice synthesis errors - CRITICAL for production
  if (message.type === 'speech-update' && message.status === 'error') {
    const session = activeSessions.get(callId);
    if (session) {
      session.voiceErrors++;
      
      // Fallback after 3 consecutive failures
      if (session.voiceErrors >= 3) {
        console.error(`ElevenLabs failing for call ${callId}. Implement fallback voice.`);
        // In production: switch to backup TTS provider
      }
    }
    console.error('ElevenLabs synthesis failed:', message.error);
  }

  // Handle barge-in with TTS cancellation
  if (message.type === 'speech-update' && message.status === 'started') {
    const session = activeSessions.get(callId);
    
    if (session?.isSpeaking) {
      session.isSpeaking = false;
      session.audioBuffer = []; // Flush buffer to prevent stale audio
      console.log(`[${callId}] Barge-in detected - TTS cancelled at ${Date.now()}`);
    }
    
    // Process partial transcript
    const partialText = message.transcript?.partial || '';
    if (partialText.length > 10) { // Ignore noise
      session.lastInterruptTime = Date.now();
      session.interruptCount = (session.interruptCount || 0) + 1;
    }
  }

  // Track latency for voice cloning - catches network issues
  if (message.type === 'transcript' && message.transcriptType === 'final') {
    const latency = Date.now() - message.timestamp;
    if (latency > 1500) {
      const session = activeSessions.get(callId);
      if (session) session.latencyWarnings++;
      console.warn(`High latency detected: ${latency}ms on call ${callId}`);
    }
  }

  // Cleanup on call end
  if (message.type === 'end-of-call-report') {
    const session = activeSessions.get(callId);
    if (session) {
      console.log(`Call ${callId} stats:`, {
        duration: Date.now() - session.startTime,
        voiceErrors: session.voiceErrors,
        latencyWarnings: session.latencyWarnings,
        interruptCount: session.interruptCount || 0
      });
      activeSessions.delete(callId);
    }
  }

  res.status(200).json({ received: true });
});

app.listen(3000, () => console.log('Webhook server running on port 3000'));

# Start ngrok tunnel
ngrok http 3000

# Test webhook locally
curl -X POST http://localhost:3000/webhook/vapi \
  -H "Content-Type: application/json" \
  -H "x-vapi-signature: $(echo -n '{"message":{"type":"assistant-request"},"call":{"id":"test-call-123"}}' | openssl dgst -sha256 -hmac "$WEBHOOK_SECRET" | awk '{print $2}')" \
  -d '{"message":{"type":"assistant-request"},"call":{"id":"test-call-123"}}'

# Expected output: {"received":true}
# Status code: 200

// Validate incoming webhook signature
function validateWebhook(payload, signature) {
  const hash = crypto
    .createHmac('sha256', process.env.WEBHOOK_SECRET)
    .update(JSON.stringify(payload))
    .digest('hex');
  
  if (hash !== signature) {
    throw new Error('Invalid webhook signature');
  }
  return true;
}

// Apply in webhook handler
app.post('/webhook/vapi', (req, res) => {
  const signature = req.headers['x-vapi-signature'];
  
  try {
    validateWebhook(req.body, signature);
    // Process webhook...
    res.status(200).json({ received: true });
  } catch (error) {
    console.error('Webhook validation failed:', error);
    res.status(401).json({ error: 'Unauthorized' });
  }
});

{
  "message": {
    "type": "speech-update",
    "status": "started",
    "timestamp": 1704067234567,
    "transcript": {
      "partial": "wait I need to update my",
      "isFinal": false
    },
    "call": {
      "id": "call_abc123",
      "status": "in-progress"
    }
  }
}