// Server setup - Express with WebSocket support for real-time updates
const express = require('express');
const WebSocket = require('ws');
const crypto = require('crypto');
const app = express();

// Production configuration with environment variables
const config = {
  vapi: {
    apiKey: process.env.VAPI_API_KEY,
    webhookSecret: process.env.VAPI_WEBHOOK_SECRET,
    baseUrl: 'https://api.vapi.ai'
  },
  twilio: {
    accountSid: process.env.TWILIO_ACCOUNT_SID,
    authToken: process.env.TWILIO_AUTH_TOKEN,
    phoneNumber: process.env.TWILIO_PHONE_NUMBER
  },
  server: {
    port: process.env.PORT || 3000,
    webhookUrl: process.env.WEBHOOK_URL // Your ngrok/production URL
  }
};

// Session state with TTL cleanup - prevents memory leaks
const activeSessions = new Map();
const SESSION_TTL = 30 * 60 * 1000; // 30 minutes

function initializeSession(sessionId) {
  const session = {
    id: sessionId,
    transcripts: [],
    isProcessing: false,
    startTime: Date.now(),
    metadata: {}
  };
  activeSessions.set(sessionId, session);
  
  // Auto-cleanup to prevent memory bloat
  setTimeout(() => {
    if (activeSessions.has(sessionId)) {
      activeSessions.delete(sessionId);
      console.log(`Session ${sessionId} expired and cleaned up`);
    }
  }, SESSION_TTL);
  
  return session;
}

// Webhook signature validation - security is not optional
function validateSignature(signature, body, secret) {
  const hmac = crypto.createHmac('sha256', secret);
  const digest = hmac.update(JSON.stringify(body)).digest('hex');
  return crypto.timingSafeEqual(
    Buffer.from(signature),
    Buffer.from(digest)
  );
}

app.use(express.json());
app.use(express.urlencoded({ extended: true }));

flowchart LR
    A[Customer Calls] --> B[Twilio Receives Call]
    B --> C[Forward to VAPI Assistant]
    C --> D[Real-time STT Stream]
    D --> E[Your Webhook Handler]
    E --> F[Process & Route]
    F --> G[TTS Response via VAPI]
    G --> H[Stream to Customer]

// Assistant config for live support - optimized for interruptions
const assistantConfig = {
  model: {
    provider: "openai",
    model: "gpt-4",
    temperature: 0.7,
    messages: [{
      role: "system",
      content: "You are a live support agent. Keep responses under 20 words. If customer interrupts, stop immediately."
    }]
  },
  voice: {
    provider: "11labs",
    voiceId: "21m00Tcm4TlvDq8ikWAM", // Rachel voice
    stability: 0.5,
    similarityBoost: 0.75,
    optimizeStreamingLatency: 3 // Max optimization for real-time
  },
  transcriber: {
    provider: "deepgram",
    model: "nova-2",
    language: "en",
    smartFormat: true,
    endpointing: 150 // Aggressive - detect silence after 150ms
  },
  recordingEnabled: true,
  firstMessage: "Hi, I'm here to help. What can I assist you with today?",
  serverUrl: config.server.webhookUrl + "/webhook/vapi",
  serverUrlSecret: config.vapi.webhookSecret
};

// Create assistant via VAPI API - this is YOUR server calling VAPI
async function createAssistant() {
  try {
    const response = await fetch(`${config.vapi.baseUrl}/assistant`, {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${config.vapi.apiKey}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify(assistantConfig)
    });
    
    if (!response.ok) {
      const error = await response.json();
      throw new Error(`VAPI API error ${response.status}: ${error.message}`);
    }
    
    const assistant = await response.json();
    console.log(`Assistant created: ${assistant.id}`);
    return assistant.id;
  } catch (error) {
    console.error('Assistant creation failed:', error);
    throw error;
  }
}

// Webhook handler - processes real-time events from VAPI
app.post('/webhook/vapi', async (req, res) => {
  const event = req.body;
  
  // Validate webhook signature - prevents replay attacks
  const signature = req.headers['x-vapi-signature'];
  if (!signature || !validateSignature(signature, req.body, config.vapi.webhookSecret)) {
    console.error('Invalid webhook signature');
    return res.status(401).json({ error: 'Invalid signature' });
  }

  const sessionId = event.call?.id;
  if (!sessionId) {
    return res.status(400).json({ error: 'Missing call ID' });
  }
  
  // Initialize session if new
  if (!activeSessions.has(sessionId)) {
    initializeSession(sessionId);
  }
  
  const session = activeSessions.get(sessionId);
  
  // Guard against race conditions - only one handler per session at a time
  if (session.isProcessing) {
    console.log(`Session ${sessionId} already processing, queuing event`);
    return res.status(200).json({ queued: true });
  }

  // Return 200 immediately - process async to avoid webhook timeout
  res.status(200).json({ received: true });
  
  // Process event asynchronously
  setImmediate(async () => {
    session.isProcessing = true;
    
    try {
      switch(event.message.type) {
        case 'transcript':
          // Handle both partial and final transcripts
          if (!event.message.transcriptType || event.message.transcriptType === 'partial') {
            await handlePartialTranscript(sessionId, event.message.transcript);
          } else if (event.message.transcriptType === 'final') {
            await handleFinalTranscript(sessionId, event.message.transcript);
          }
          break;
          
        case 'function-call':
          // Customer needs escalation or specific action
          await handleFunctionCall(sessionId, event.message);
          break;
          
        case 'speech-update':
          // Real-time speech status for UI indicators
          await handleSpeechUpdate(sessionId, event.message);
          break

### System Diagram

System architecture for vapi integration with your application.

```mermaid
graph TB
    User[User Device]
    VAPI[VAPI Service]
    ASR[Automatic Speech Recognition]
    NLP[NLP Processor]
    TTS[Text-to-Speech Engine]
    Webhook[Webhook Server]
    DB[(Database)]
    ErrorHandler[Error Handler]

    User-->|Voice Input|VAPI
    VAPI-->ASR
    ASR-->NLP
    NLP-->TTS
    TTS-->|Voice Output|User
    VAPI-->|Event Data|Webhook
    Webhook-->DB
    VAPI-->|Error Events|ErrorHandler
    ErrorHandler-->|Log|DB
    NLP-->|Error|ErrorHandler
    ASR-->|Error|ErrorHandler
    TTS-->|Error|ErrorHandler

// Test WebSocket connection and session handling
const WebSocket = require('ws');

const ws = new WebSocket('ws://localhost:3000');
const testSessionId = crypto.randomBytes(16).toString('hex');

ws.on('open', () => {
  console.log('WebSocket connected');
  
  // Simulate VAPI session start
  ws.send(JSON.stringify({
    type: 'session.start',
    sessionId: testSessionId,
    timestamp: Date.now()
  }));
  
  // Test partial transcript handling
  setTimeout(() => {
    ws.send(JSON.stringify({
      type: 'transcript.partial',
      sessionId: testSessionId,
      text: 'I need help with my account'
    }));
  }, 1000);
});

ws.on('message', (data) => {
  const message = JSON.parse(data);
  console.log('Server response:', message);
  
  // Verify session exists in activeSessions
  if (message.type === 'session.created') {
    console.log('✓ Session initialized:', message.sessionId);
  }
});

ws.on('error', (error) => {
  console.error('WebSocket error:', error.message);
});

// Test webhook signature validation locally
const testPayload = {
  type: 'call.started',
  sessionId: testSessionId,
  timestamp: Date.now()
};

const testSignature = crypto
  .createHmac('sha256', config.server.webhookSecret)
  .update(JSON.stringify(testPayload))
  .digest('hex');

console.log('Expected signature:', testSignature);

// Send test webhook with curl
// curl -X POST http://localhost:3000/webhook \
//   -H "x-vapi-signature: <testSignature>" \
//   -H "Content-Type: application/json" \
//   -d '<testPayload>'

// Barge-in handler with buffer flush
const handleInterruption = async (sessionId, partialTranscript) => {
  const session = activeSessions.get(sessionId);
  if (!session) return;

  // Race condition guard - prevent multiple interrupts
  if (session.isProcessing) {
    console.log(`[${sessionId}] Already processing, queuing interrupt`);
    session.pendingInterrupt = partialTranscript;
    return;
  }

  session.isProcessing = true;

  try {
    // Flush TTS buffer immediately
    if (session.audioBuffer && session.audioBuffer.length > 0) {
      console.log(`[${sessionId}] Flushing ${session.audioBuffer.length} audio chunks`);
      session.audioBuffer = [];
    }

    // Cancel ongoing TTS request
    if (session.ttsController) {
      session.ttsController.abort();
      session.ttsController = null;
    }

    // Send interrupt signal to VAPI
    const response = await fetch('https://api.vapi.ai/chat', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.VAPI_API_KEY}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        assistantId: session.assistantId,
        message: {
          type: 'interrupt',
          transcript: partialTranscript,
          timestamp: Date.now()
        }
      })
    });

    if (!response.ok) {
      throw new Error(`Interrupt failed: ${response.status}`);
    }

    const result = await response.json();
    session.lastInterruptAt = Date.now();
    
  } catch (error) {
    console.error(`[${sessionId}] Interrupt handling failed:`, error);
  } finally {
    session.isProcessing = false;
    
    // Process queued interrupt if exists
    if (session.pendingInterrupt) {
      const queued = session.pendingInterrupt;
      session.pendingInterrupt = null;
      await handleInterruption(sessionId, queued);
    }
  }
};

// Edge case: Stale interrupt detection
if (Date.now() - session.lastInterruptAt < 500) {
  console.log(`[${sessionId}] Ignoring stale interrupt (${Date.now() - session.lastInterruptAt}ms old)`);
  return;
}

// WRONG: No cancellation logic
ws.on('message', (data) => {
  const event = JSON.parse(data);
  if (event.type === 'transcript-partial') {
    generateTTSResponse(event.text); // Queues audio without checking state
  }
});

// CORRECT: Cancel in-flight TTS on barge-in
let currentTTSStream = null;

ws.on('message', async (data) => {
  const event = JSON.parse(data);
  
  if (event.type === 'speech-start') {
    // User started speaking - kill active TTS immediately
    if (currentTTSStream) {
      currentTTSStream.abort();
      currentTTSStream = null;
    }
  }
  
  if (event.type === 'transcript-final') {
    const controller = new AbortController();
    currentTTSStream = controller;
    
    try {
      await fetch('https://api.elevenlabs.io/v1/text-to-speech/' + config.voice.voiceId, {
        method: 'POST',
        headers: {
          'xi-api-key': process.env.ELEVENLABS_API_KEY,
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({ text: event.text }),
        signal: controller.signal
      });
    } catch (error) {
      if (error.name === 'AbortError') return; // Expected on interruption
      console.error('TTS Error:', error);
    }
  }
});

const PING_INTERVAL = 20000; // 20s - below carrier timeout thresholds

ws.on('open', () => {
  const session = activeSessions.get(testSessionId);
  
  session.pingTimer = setInterval(() => {
    if (ws.readyState === WebSocket.OPEN) {
      ws.ping();
    } else {
      clearInterval(session.pingTimer);
    }
  }, PING_INTERVAL);
});

ws.on('pong', () => {
  const session = activeSessions.get(testSessionId);
  session.lastPong = Date.now();
});

ws.on('close', () => {
  const session = activeSessions.get(testSessionId);
  if (session?.pingTimer) clearInterval(session.pingTimer);
});

const assistantConfig = {
  transcriber: {
    provider: 'deepgram',
    model: 'nova-2-phonecall', // Optimized for telephony
    language: 'en',
    encoding: 'mulaw', // CRITICAL: Must match Twilio's codec
    sampleRate: 8000,  // Twilio's native rate
    endpointing: 800   // Longer for phone latency
  }
};

const express = require('express');
const WebSocket = require('ws');
const crypto = require('crypto');
const fetch = require('node-fetch');

const app = express();
app.use(express.json());

// Configuration from previous sections
const config = {
  vapi: {
    apiKey: process.env.VAPI_API_KEY,
    baseUrl: 'https://api.vapi.ai'
  },
  twilio: {
    accountSid: process.env.TWILIO_ACCOUNT_SID,
    authToken: process.env.TWILIO_AUTH_TOKEN
  },
  server: {
    port: process.env.PORT || 3000,
    webhookSecret: process.env.WEBHOOK_SECRET
  }
};

// Session management with TTL
const activeSessions = new Map();
const SESSION_TTL = 30 * 60 * 1000; // 30 minutes

function initializeSession(sessionId) {
  const session = {
    id: sessionId,
    transcripts: [],
    metadata: { startTime: Date.now() },
    isProcessing: false,
    currentTTSStream: null
  };
  activeSessions.set(sessionId, session);
  
  // Auto-cleanup to prevent memory leaks
  setTimeout(() => {
    if (activeSessions.has(sessionId)) {
      const session = activeSessions.get(sessionId);
      if (session.currentTTSStream) {
        session.currentTTSStream.abort();
      }
      activeSessions.delete(sessionId);
      console.log(`Session ${sessionId} expired and cleaned up`);
    }
  }, SESSION_TTL);
  
  return session;
}

// Webhook signature validation (security is not optional)
function validateSignature(payload, signature) {
  const hmac = crypto.createHmac('sha256', config.server.webhookSecret);
  hmac.update(JSON.stringify(payload));
  const digest = hmac.digest('hex');
  return crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(digest));
}

// Create assistant with streaming configuration
async function createAssistant() {
  const assistantConfig = {
    model: {
      provider: "openai",
      model: "gpt-4",
      temperature: 0.7,
      messages: [{
        role: "system",
        content: "You are a live support agent. Keep responses under 30 words. Handle interruptions gracefully."
      }]
    },
    voice: {
      provider: "elevenlabs",
      voiceId: "21m00Tcm4TlvDq8ikWAM",
      stability: 0.5,
      similarityBoost: 0.75,
      optimizeStreamingLatency: 2 // Critical for low-latency TTS integration
    },
    transcriber: {
      provider: "deepgram",
      model: "nova-2",
      language: "en",
      endpointing: 200 // Barge-in detection threshold
    },
    firstMessage: "Hi, I'm your support agent. How can I help?"
  };

  try {
    const response = await fetch(`${config.vapi.baseUrl}/assistant`, {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${config.vapi.apiKey}`,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify(assistantConfig)
    });

    if (!response.ok) {
      const error = await response.json();
      throw new Error(`Assistant creation failed: ${error.message}`);
    }

    const assistant = await response.json();
    console.log(`Assistant created: ${assistant.id}`);
    return assistant;
  } catch (error) {
    console.error('Failed to create assistant:', error);
    throw error;
  }
}

// WebSocket handler for VAPI WebSocket streaming
const wss = new WebSocket.Server({ noServer: true });

wss.on('connection', (ws, request) => {
  const sessionId = request.url.split('/').pop();
  const session = initializeSession(sessionId);
  
  console.log(`WebSocket connected: ${sessionId}`);

  // Handle streaming transcripts
  ws.on('message', (data) => {
    try {
      const message = JSON.parse(data);
      
      if (message.type === 'transcript-partial') {
        // Real-time voice transcription - update UI immediately
        session.transcripts.push({
          type: 'partial',
          text: message.text,
          timestamp: Date.now()
        });
      }
      
      if (message.type === 'transcript-final') {
        // Replace partial with final transcript
        session.transcripts = session.transcripts.filter(t => t.type !== 'partial');
        session.transcripts.push({
          type: 'final',
          text: message.text,
          timestamp: Date.now()
        });
      }

      if (message.type === 'interruption') {
        // Handle barge-in: cancel current TTS stream
        if (session.currentTTSStream && !session.currentTTSStream.aborted) {
          session.currentTTSStream.abort();
          session.currentTTSStream = null;
          console.log(`TTS stream cancelled for session ${sessionId}`);
        }
      }

      // Broadcast to all connected clients (Server-Sent Events pattern)
      ws.send(JSON.stringify({
        sessionId: session.id,
        transcripts: session.transcripts,
        metadata: session.metadata
      }));

    } catch (error) {
      console.error('WebSocket message error:', error);
    }
  });

  ws.on('close', () => {
    console.log(`WebSocket disconnected: ${sessionId}`);
  });

  // Keep-alive ping to prevent connection drops
  const PING_INTERVAL = setInterval(() => {
    if (ws.readyState === WebSocket.OPEN) {
      ws.ping();
    }
  }, 30000);

  ws.on('close', () => clearInterval(PING_INTERVAL));
});

// Webhook endpoint for VAPI events
app.post('/webhook/vapi', async (req, res) => {
  const signature = req.headers['x-vapi-signature'];
  
  if (!validateSignature(req.body, signature)) {
    console.error('Invalid webhook signature');
    return res.status(401).json({ error: 'Invalid signature' });
  }

  const event = req.body;
  const sessionId = event.call?.id || event.message?.call?.id;

  if (!sessionId) {
    return res.status(400).json({ error: 'Missing session ID' });
  }

  const session = activeSessions.get(sessionId);
  if (!session) {
    console.warn(`Received event for unknown session: ${sessionId}`);
    return res.status(404).json({ error: 'Session not found' });
  }

  // Handle different event types
  switch (event.type) {
    case 'call-started':
      session.metadata.callStarted = Date.now();
      break;
      
    case 'speech-started':
      session.isProcessing = true;
      break;
      
    case 'speech-ended':
      session.isProcessing = false;
      break;
      
    case 'call-ended':
      session.metadata.callEnded = Date.now();
      session.metadata.duration = session.metadata.callEnded - session.metadata.callStarted;
      break;
  }

  res.status

## FAQ

### Technical Questions

**How does VAPI handle WebSocket streaming for real-time voice transcription?**

VAPI maintains a persistent WebSocket connection that receives audio chunks (typically PCM 16kHz, 16-bit) from the client and streams partial transcripts back in real-time. The transcriber processes audio frames asynchronously, emitting `transcript.partial` events before the final `transcript.final` event fires. This dual-event pattern lets you display live captions while the user is still speaking. The key is buffering incoming audio chunks in a queue (not dropping frames) and processing them sequentially to avoid race conditions between partial and final transcripts.

**What's the latency impact of adding Twilio integration to VAPI?**

Twilio adds ~200-400ms of additional latency due to SIP signaling and media gateway routing. VAPI's native latency is ~150-300ms (STT + LLM + TTS). Combined, expect 350-700ms end-to-end for a user utterance to trigger a bot response. Mitigate this by: (1) enabling `optimizeStreamingLatency: true` in the TTS config to stream audio chunks instead of waiting for full synthesis, (2) using partial transcripts to start LLM processing before the user finishes speaking, (3) reducing model inference time by using smaller models (gpt-3.5-turbo vs gpt-4).

**Why does my bot interrupt the user mid-sentence?**

VAD (Voice Activity Detection) threshold is too aggressive. Default endpointing fires after 500-800ms of silence, but network jitter can trigger false positives. Increase the `endpointing` threshold to 1200-1500ms and set `silenceThreshold` to 0.5+ (default 0.3) to reduce breathing-sound false triggers. Also check if your Twilio SIP trunk is dropping RTP packets—packet loss causes the transcriber to misinterpret silence as speech end.

### Performance

**How do I prevent TTS buffer overflow during rapid exchanges?**

Implement a cancellation controller: when a new user message arrives, abort the current TTS stream immediately using `AbortController`. Store the active stream reference in `currentTTSStream` and call `controller.abort()` before queuing new audio. Without this, old TTS audio queues up and plays after the bot has already moved to the next response, creating overlapping speech.

**What's the maximum concurrent sessions VAPI can handle?**

VAPI's free tier supports ~10 concurrent calls; paid tiers scale to 100+ depending on your plan. Per-session memory usage is ~2-5MB (transcript history + session state). If you're storing `activeSessions` in-memory, set `SESSION_TTL` to 3600 seconds (1 hour) and implement cleanup: `setTimeout(() => delete activeSessions[sessionId], SESSION_TTL * 1000)`. For production, use Redis instead of in-memory storage.

### Platform Comparison

**Should I use VAPI's native voice or Twilio's voice synthesis?**

Use VAPI's native voice (ElevenLabs or Google). Twilio's voice synthesis is older (lower naturalness) and adds extra latency. Configure VAPI's `voice.provider: "elevenlabs"` with `optimizeStreamingLatency: true` to stream audio chunks. Twilio's role is media routing only—let VAPI own the voice experience.

**Can I replace Twilio with a different SIP provider?**

Yes, but Twilio is the easiest integration. Other SIP providers (Vonage, Bandwidth) work, but you'll need to handle SIP registration, media negotiation, and RTP routing yourself. Stick with Twilio unless you have specific cost or compliance requirements (e.g., HIPAA-compliant media gateways).

## Resources

**Twilio**: Get Twilio Voice API → [https://www.twilio.com/try-twilio](https://www.twilio.com/try-twilio)

**VAPI Documentation**
- [VAPI API Reference](https://docs.vapi.ai) – WebSocket streaming, real-time voice transcription, low-latency TTS integration
- [VAPI Assistant Configuration](https://docs.vapi.ai/assistants) – Model, voice, transcriber setup for bidirectional audio

**Twilio Integration**
- [Twilio Voice API](https://www.twilio.com/docs/voice) – SIP trunking, call routing for live support
- [Twilio WebSocket Connections](https://www.twilio.com/docs/voice/twiml/stream) – Server-Sent Events (SSE) patterns

**Implementation References**
- [VAPI GitHub Examples](https://github.com/VapiAI) – Production streaming implementations
- [Express WebSocket Middleware](https://github.com/websockets/ws) – Real-time bidirectional communication

## References

1. https://docs.vapi.ai/chat/quickstart
2. https://docs.vapi.ai/quickstart/web
3. https://docs.vapi.ai/workflows/quickstart
4. https://docs.vapi.ai/quickstart/phone
5. https://docs.vapi.ai/server-url/developing-locally
6. https://docs.vapi.ai/quickstart/introduction
7. https://docs.vapi.ai/observability/evals-quickstart
8. https://docs.vapi.ai/assistants/structured-outputs-quickstart
9. https://docs.vapi.ai/tools/custom-tools