Integrate Node.js with Retell AI and Twilio: Lessons from My Setup

sequenceDiagram
    participant Caller
    participant Twilio
    participant NodeServer
    participant RetellAI
    Caller->>Twilio: Initiates call
    Twilio->>NodeServer: POST /webhook/twilio-incoming
    NodeServer->>RetellAI: Register call (agent_id, audio config)
    RetellAI->>NodeServer: WebSocket URL
    NodeServer->>Twilio: TwiML with <Stream> to WebSocket
    Twilio->>NodeServer: WebSocket connection (media events)
    NodeServer->>RetellAI: Forward audio chunks (mulaw, 8kHz)
    RetellAI->>NodeServer: AI response audio
    NodeServer->>Twilio: Stream response back
    Twilio->>Caller: Audio playback
    Caller->>Twilio: Ends call
    Twilio->>NodeServer: WebSocket close
    NodeServer->>RetellAI: Disconnect
    RetellAI->>NodeServer: POST /webhook/retell-events (call_ended)

// .env file
TWILIO_ACCOUNT_SID=ACxxxxx
TWILIO_AUTH_TOKEN=your_auth_token
TWILIO_PHONE_NUMBER=+1234567890
RETELL_API_KEY=key_xxxxx
RETELL_AGENT_ID=agent_xxxxx
SERVER_URL=https://your-domain.ngrok.io
PORT=3000

const express = require('express');
const twilio = require('twilio');
const { RetellClient } = require('@retellai/retell-sdk');

const app = express();
app.use(express.urlencoded({ extended: false }));

const retellClient = new RetellClient({
  apiKey: process.env.RETELL_API_KEY
});

// Validate Twilio signature before processing
function validateTwilioSignature(req, res, next) {
  const signature = req.headers['x-twilio-signature'];
  const url = `${process.env.SERVER_URL}${req.originalUrl}`;
  
  if (!twilio.validateRequest(process.env.TWILIO_AUTH_TOKEN, signature, url, req.body)) {
    return res.status(403).send('Forbidden');
  }
  next();
}

app.post('/webhook/twilio-incoming', validateTwilioSignature, async (req, res) => {
  const twiml = new twilio.twiml.VoiceResponse();
  
  try {
    // Register call with Retell to get WebSocket URL (must complete in <2s)
    const retellCall = await Promise.race([
      retellClient.call.register({
        agent_id: process.env.RETELL_AGENT_ID,
        audio_websocket_protocol: "twilio",
        audio_encoding: "mulaw", // Twilio's audio format
        sample_rate: 8000 // Twilio uses 8kHz
      }),
      new Promise((_, reject) => setTimeout(() => reject(new Error('Timeout')), 2000))
    ]);

    // Connect Twilio call to Retell's WebSocket
    const connect = twiml.connect();
    connect.stream({
      url: retellCall.call_detail.websocket_url
    });

    res.type('text/xml');
    res.send(twiml.toString());
  } catch (error) {
    console.error('Retell registration failed:', error);
    twiml.say('Sorry, the system is unavailable. Please try again later.');
    res.type('text/xml');
    res.send(twiml.toString());
  }
});

app.post('/webhook/retell-events', express.json(), (req, res) => {
  const event = req.body;

  switch(event.event) {
    case 'call_started':
      console.log(`Call ${event.call.call_id} started at ${event.call.start_timestamp}`);
      // Initialize session state, log to analytics
      break;
    
    case 'call_ended':
      const duration = event.call.end_timestamp - event.call.start_timestamp;
      console.log(`Call ${event.call.call_id} ended. Duration: ${duration}ms`);
      // Save transcript to database, calculate API costs
      break;
    
    case 'call_analyzed':
      // Post-call analysis with sentiment scores, summary
      console.log('Analysis:', event.call.call_analysis);
      break;
  }

  res.sendStatus(200); // Always return 200 or Retell retries with exponential backoff
});

const WebSocket = require('ws');

const wss = new WebSocket.Server({ noServer: true });

wss.on('connection', (ws, req) => {
  let retellWs = null;
  let streamSid = null;
  let isProcessingAudio = false;

  // Connect to Retell AI's WebSocket
  const retellUrl = `wss://api.retellai.com/audio-websocket/${process.env.RETELL_AGENT_ID}`;
  retellWs = new WebSocket(retellUrl, {
    headers: { 'Authorization': `Bearer ${process.env.RETELL_API_KEY}` }
  });

  retellWs.on('open', () => {
    retellWs.send(JSON.stringify({
      type: 'config',
      config: {
        agent_id: process.env.RETELL_AGENT_ID,
        audio_encoding: 'mulaw',
        sample_rate: 8000
      }
    }));
  });

  // Twilio → Retell: Forward caller audio
  ws.on('message', async (message) => {
    const event = JSON.parse(message);

    if (event.event === 'start') {
      streamSid = event.start.streamSid;
    }

    if (event.event === 'media' && retellWs.readyState === WebSocket.OPEN) {
      // Guard against race conditions with processing flag
      if (isProcessingAudio) {
        return; // Drop frame instead of queuing
      }
      
      isProcessingAudio = true;
      
      try {
        retellWs.send(JSON.stringify({
          type: 'audio',
          audio: event.media.payload // Base64 mulaw
        }));
      } finally {
        isProcessingAudio = false;
      }
    }

    if (event.event === 'stop') {
      retellWs.close();
    }
  });

  // Retell → Twilio: Stream agent responses back
  retellWs.on('message', (data) => {
    const payload = JSON.parse(data);

    if (payload.type === 'audio' && ws.readyState === WebSocket.OPEN) {
      ws.send(JSON.stringify({
        event: 'media',
        streamSid: streamSid,
        media: { payload: payload.audio }
      }));
    }
  });

  ws.on('close', () => {
    if (retellWs) retellWs.close();
  });
});

// Upgrade HTTP to WebSocket
const server = app.listen(process.env.PORT, () => {
  console.log(`Server running on port ${process.env.PORT}`);
  console.log(`Expose with: ngrok http ${process.env.PORT}`);
  console.log(`Set Twilio webhook to: ${process.env.SERVER_URL}/webhook/twilio-incoming`);
});

server.on('upgrade', (request, socket, head) => {
  wss.handleUpgrade(request, socket, head, (ws) => {
    wss.emit('connection', ws, request);
  });
});

// retellAgentConfig.js
module.exports = {
  agent_id: process.env.RETELL_AGENT_ID,
  agent_name: "Customer Support Agent",
  voice_id: "11labs-Adrian", // ElevenLabs voice (alternatives: "openai-alloy", "deepgram-aura")
  language: "en-US",
  response_engine: {
    type: "retell-llm",
    llm_id: process.env.RETELL_LLM_ID, // GPT-4 for accuracy, GPT-3.5 for speed
    temperature: 0.7 // 0.0-1.0, higher = more creative but less predictable
  },
  begin_message: "Thanks for calling. How can I help you today?",
  general_prompt: "You are a helpful customer support agent. Be concise and professional. If you don't know something, say so instead of guessing.",
  enable_backchannel: true, // Agent says "mm-hmm" during user speech for natural feel
  ambient_sound: "office", // Options: "off", "coffee_shop", "office"
  interruption_sensitivity: 0.7, // 0-1 scale, higher = easier to interrupt (0.5-0.8 recommended)
  audio_websocket_protocol: "twilio",
  audio_encoding: "mulaw", // Must match Twilio's format
  sample_rate: 8000, // Twilio default, don't change unless transcoding
  end_call_after_silence_ms: 30000, // Hang up after 30s of silence
  max_call_duration_ms: 600000, // 10-minute hard limit to prevent runaway costs
  webhook_url: `${process.env.SERVER_URL}/webhook/retell-events`, // Where Retell sends call events
  fallback_voice_ids: ["11labs-Rachel", "openai-nova"] // Backup voices if primary fails
};

# Start your Express server
node server.js

# In another terminal, expose it
ngrok http 3000

# Copy the HTTPS URL (e.g., https://abc123.ngrok.io)
# Update Twilio webhook in console: https://abc123.ngrok.io/webhook/twilio-incoming

# Simulate Twilio's incoming call webhook
curl -X POST https://abc123.ngrok.io/webhook/twilio-incoming \
  -d "CallSid=CA1234567890abcdef" \
  -d "From=+15555551234" \
  -d "To=+15555556789" \
  -H "X-Twilio-Signature: fake_signature_for_testing"

# Expected response (TwiML):
<?xml version="1.0" encoding="UTF-8"?>
<Response>
  <Connect>
    <Stream url="wss://api.retellai.com/audio-websocket/agent_xxxxx"/>
  </Connect>
</Response>

// t=0ms (2:34:17.000 PM): Call arrives
{
  event: 'call_started',
  call: {
    call_id: 'call_abc123',
    from_number: '+15555551234',
    to_number: '+15555556789',
    start_timestamp: 1704139457000
  }
}

// t=340ms: Agent starts greeting
// Agent: "Thanks for calling Acme Support. How can I—"

// t=1200ms: User interrupts
// User: "I need to speak to a human."
{
  event: 'transcript',
  transcript: {
    role: 'user',
    content: 'I need to speak to a human.',
    timestamp: 1704139458200
  }
}

// t=1220ms: Agent stops mid-sentence (barge-in detected)
// Audio buffer flushed, TTS cleared

// t=1800ms: Agent responds
// Agent: "I understand. Let me connect you to our support team."
{
  event: 'transcript',
  transcript: {
    role: 'agent',
    content: 'I understand. Let me connect you to our support team.',
    timestamp: 1704139458800
  }
}

// t=4500ms: User interrupts again
// User: "Wait, actually, can you just reset my password?"

// t=4520ms: Agent stops, processes new request
{
  event: 'transcript',
  transcript: {
    role: 'user',
    content: 'Wait, actually, can you just reset my password?',
    timestamp: 1704139461520
  }
}

// t=6200ms: Agent provides password reset instructions
// Agent: "Sure, I've sent a reset link to your email. Check your inbox."

// t=12000ms: Call ends
{
  event: 'call_ended',
  call: {
    call_id: 'call_abc123',
    end_timestamp: 1704139469000,
    duration_ms: 12000,
    transcript: [...], // Full conversation
    call_analysis: {
      sentiment: 'neutral',
      summary: 'User requested password reset, provided via email.',
      resolution: 'resolved'
    }
  }
}