flowchart LR
    A[Microphone] -->|WebSocket| B[Deepgram STT]
    B -->|Transcript| C[LLM Processing]
    C -->|Response Text| D[PlayHT TTS]
    D -->|Audio Stream| E[Speaker]
    E -.->|Barge-in Signal| B

const deepgramConfig = {
  model: 'nova-2',
  language: 'en-US',
  encoding: 'linear16',
  sample_rate: 16000,
  channels: 1,
  interim_results: true,
  endpointing: 300,  // 300ms silence = end of utterance
  vad_events: true,  // Voice activity detection
  punctuate: true
};

const playhtConfig = {
  voice: 'larry',  // Or your cloned voice ID
  output_format: 'mp3',
  sample_rate: 24000,
  quality: 'medium',  // Balance latency vs quality
  speed: 1.0,
  seed: null  // Randomize for natural variation
};

const deepgramWs = new WebSocket(
  `wss://api.deepgram.com/v1/listen?${new URLSearchParams(deepgramConfig)}`,
  { headers: { 'Authorization': `Token ${process.env.DEEPGRAM_API_KEY}` }}
);

// PlayHT uses HTTP streaming, not WebSocket
const playhtStream = await fetch('https://api.play.ht/api/v2/tts/stream', {
  method: 'POST',
  headers: {
    'Authorization': `Bearer ${process.env.PLAYHT_API_KEY}`,
    'X-User-ID': process.env.PLAYHT_USER_ID,
    'Content-Type': 'application/json'
  },
  body: JSON.stringify({
    text: responseText,
    voice: playhtConfig.voice,
    output_format: playhtConfig.output_format,
    sample_rate: playhtConfig.sample_rate
  })
});

let isProcessing = false;  // Race condition guard

navigator.mediaDevices.getUserMedia({ audio: true })
  .then(stream => {
    const mediaRecorder = new MediaRecorder(stream, {
      mimeType: 'audio/webm;codecs=opus'
    });
    
    mediaRecorder.ondataavailable = (event) => {
      if (deepgramWs.readyState === WebSocket.OPEN) {
        deepgramWs.send(event.data);
      }
    };
    
    mediaRecorder.start(250);  // Send chunks every 250ms
  });

deepgramWs.onmessage = async (message) => {
  const data = JSON.parse(message.data);
  
  if (data.is_final) {
    const transcript = data.channel.alternatives[0].transcript;
    
    if (isProcessing) return;  // Prevent race condition
    isProcessing = true;
    
    try {
      const llmResponse = await processWithLLM(transcript);
      await streamTTSResponse(llmResponse);
    } finally {
      isProcessing = false;
    }
  }
};

async function streamTTSResponse(text) {
  const response = await fetch('https://api.play.ht/api/v2/tts/stream', {
    method: 'POST',
    headers: {
      'Authorization': `Bearer ${process.env.PLAYHT_API_KEY}`,
      'X-User-ID': process.env.PLAYHT_USER_ID,
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ text, ...playhtConfig })
  });
  
  const reader = response.body.getReader();
  const audioContext = new AudioContext({ sampleRate: 24000 });
  
  while (true) {
    const { done, value } = await reader.read();
    if (done) break;
    
    // Decode and play chunk immediately
    const audioBuffer = await audioContext.decodeAudioData(value.buffer);
    const source = audioContext.createBufferSource();
    source.buffer = audioBuffer;
    source.connect(audioContext.destination);
    source.start();
  }
}

let reconnectAttempts = 0;
const maxReconnectDelay = 30000;

deepgramWs.onclose = () => {
  const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), maxReconnectDelay);
  setTimeout(() => {
    reconnectAttempts++;
    initializeDeepgram();
  }, delay);
};

deepgramWs.onmessage = (message) => {
  const data = JSON.parse(message.data);
  
  if (data.type === 'SpeechStarted') {
    // User started speaking - cancel TTS immediately
    audioContext.close();  // Stops all audio sources
    audioContext = new AudioContext({ sampleRate: 24000 });
    isProcessing = false;  // Allow new processing
  }
};

graph LR
    AudioInput[Audio Input]
    PreProcessor[Pre-Processor]
    FeatureExtractor[Feature Extraction]
    AcousticModel[Acoustic Model]
    LanguageModel[Language Model]
    Decoder[Decoder]
    PostProcessor[Post-Processor]
    Transcript[Transcript Output]
    ErrorHandler[Error Handler]
    Log[Logging]

    AudioInput-->PreProcessor
    PreProcessor-->FeatureExtractor
    FeatureExtractor-->AcousticModel
    AcousticModel-->LanguageModel
    LanguageModel-->Decoder
    Decoder-->PostProcessor
    PostProcessor-->Transcript

    PreProcessor-- Error -->ErrorHandler
    FeatureExtractor-- Error -->ErrorHandler
    AcousticModel-- Error -->ErrorHandler
    LanguageModel-- Error -->ErrorHandler
    Decoder-- Error -->ErrorHandler

    ErrorHandler-->Log
    Log-->PreProcessor

// Test STT → LLM → TTS pipeline with mock audio
async function testVoicePipeline() {
  const testAudioFile = './test_audio.wav'; // 16kHz PCM audio
  const audioData = fs.readFileSync(testAudioFile);
  
  // 1. Test Deepgram STT
  deepgramWs.send(audioData);
  deepgramWs.on('message', async (data) => {
    const transcript = JSON.parse(data).channel.alternatives[0].transcript;
    console.log('STT Output:', transcript);
    
    // 2. Test LLM response
    const llmResponse = await fetch('https://api.openai.com/v1/chat/completions', {
      method: 'POST',
      headers: { 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}` },
      body: JSON.stringify({ messages: [{ role: 'user', content: transcript }] })
    });
    
    // 3. Test PlayHT TTS
    await streamTTSResponse(llmResponse.choices[0].message.content);
  });
}

let currentAudioSource = null;
let isPlaying = false;

// Barge-in detection from Deepgram STT
deepgramWs.on('message', (message) => {
  const data = JSON.parse(message);
  
  if (data.is_final && data.speech_final) {
    const transcript = data.channel.alternatives[0].transcript;
    
    // User spoke while AI was talking = barge-in
    if (isPlaying && transcript.length > 0) {
      console.log('[BARGE-IN] User interrupted:', transcript);
      
      // CRITICAL: Stop current audio immediately
      if (currentAudioSource) {
        currentAudioSource.stop(0);
        currentAudioSource = null;
      }
      
      // Flush PlayHT stream buffer
      if (playhtStream && !playhtStream.destroyed) {
        playhtStream.destroy();
      }
      
      isPlaying = false;
      
      // Process new user input
      handleUserInput(transcript);
    }
  }
});

// Track audio playback state
function playAudioChunk(audioBuffer) {
  const source = audioContext.createBufferSource();
  source.buffer = audioBuffer;
  source.connect(audioContext.destination);
  
  currentAudioSource = source;
  isPlaying = true;
  
  source.onended = () => {
    isPlaying = false;
    currentAudioSource = null;
  };
  
  source.start(0);
}

// BROKEN: Chunks overlap because we don't track playback
playhtStream.on('data', (chunk) => {
  const audioBuffer = audioContext.decodeAudioData(chunk);
  const source = audioContext.createBufferSource();
  source.buffer = audioBuffer;
  source.connect(audioContext.destination);
  source.start(0); // ❌ Always starts immediately = overlap
});

// FIXED: Track playback timing
let nextStartTime = audioContext.currentTime;

playhtStream.on('data', async (chunk) => {
  const audioBuffer = await audioContext.decodeAudioData(chunk);
  const source = audioContext.createBufferSource();
  source.buffer = audioBuffer;
  source.connect(audioContext.destination);
  
  // Schedule next chunk after current one finishes
  source.start(Math.max(0, nextStartTime));
  nextStartTime = Math.max(audioContext.currentTime, nextStartTime) + audioBuffer.duration;
});

// Exponential backoff with jitter
async function reconnectDeepgram() {
  const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), maxReconnectDelay);
  const jitter = Math.random() * 1000; // Prevent thundering herd
  
  await new Promise(resolve => setTimeout(resolve, delay + jitter));
  
  deepgramWs = new WebSocket('wss://api.deepgram.com/v1/listen', {
    headers: { Authorization: `Token ${process.env.DEEPGRAM_API_KEY}` }
  });
  
  reconnectAttempts++;
}

deepgramWs.on('message', (data) => {
  const transcript = JSON.parse(data);
  
  if (transcript.is_final && isPlaying) {
    // Stop current audio immediately
    if (currentAudioSource) {
      currentAudioSource.stop();
      currentAudioSource = null;
    }
    
    // Clear queued chunks
    nextStartTime = audioContext.currentTime;
    isPlaying = false;
  }
});

// server.js - Production voice stack server
const WebSocket = require('ws');
const express = require('express');
const { createClient } = require('@deepgram/sdk');
const fetch = require('node-fetch');

const app = express();
const server = require('http').createServer(app);
const wss = new WebSocket.Server({ server });

// Configuration from previous sections
const deepgramConfig = {
  model: 'nova-2',
  language: 'en-US',
  encoding: 'linear16',
  sample_rate: 16000,
  channels: 1,
  endpointing: 300,
  interim_results: true
};

const playhtConfig = {
  voice: 'jennifer',
  output_format: 'mp3',
  quality: 'high',
  speed: 1.0
};

// Session state management
const sessions = new Map();
const SESSION_TTL = 300000; // 5 minutes

// Audio buffer management to prevent jitter
class AudioBuffer {
  constructor() {
    this.chunks = [];
    this.isPlaying = false;
    this.nextStartTime = 0;
  }

  add(chunk) {
    this.chunks.push(chunk);
    if (!this.isPlaying) this.play();
  }

  async play() {
    this.isPlaying = true;
    while (this.chunks.length > 0) {
      const chunk = this.chunks.shift();
      const now = Date.now();
      const delay = Math.max(0, this.nextStartTime - now);
      await new Promise(resolve => setTimeout(resolve, delay));
      // Send chunk to client
      this.nextStartTime = Date.now() + (chunk.duration || 100);
    }
    this.isPlaying = false;
  }

  clear() {
    this.chunks = [];
    this.isPlaying = false;
  }
}

// Deepgram connection with reconnection logic
function createDeepgramConnection(sessionId) {
  const deepgram = createClient(process.env.DEEPGRAM_API_KEY);
  const connection = deepgram.listen.live(deepgramConfig);
  
  const session = sessions.get(sessionId);
  session.reconnectAttempts = 0;
  const maxReconnectDelay = 30000;

  connection.on('open', () => {
    console.log(`[${sessionId}] Deepgram connected`);
    session.reconnectAttempts = 0;
  });

  connection.on('Results', async (data) => {
    if (!data.channel?.alternatives?.[0]) return;
    
    const transcript = data.channel.alternatives[0].transcript;
    if (!transcript || data.is_final === false) return;

    // Prevent race condition during TTS playback
    if (session.isProcessing) {
      console.log(`[${sessionId}] Dropped transcript (already processing)`);
      return;
    }
    session.isProcessing = true;

    try {
      // Generate LLM response (simplified - use your LLM here)
      const llmResponse = await generateResponse(transcript);
      
      // Stream TTS from PlayHT
      await streamTTSResponse(sessionId, llmResponse);
    } catch (error) {
      console.error(`[${sessionId}] Processing error:`, error);
      session.ws.send(JSON.stringify({ 
        type: 'error', 
        message: 'Processing failed' 
      }));
    } finally {
      session.isProcessing = false;
    }
  });

  connection.on('error', (error) => {
    console.error(`[${sessionId}] Deepgram error:`, error);
  });

  connection.on('close', () => {
    console.log(`[${sessionId}] Deepgram closed`);
    if (sessions.has(sessionId)) {
      reconnectDeepgram(sessionId);
    }
  });

  return connection;
}

// Exponential backoff reconnection
function reconnectDeepgram(sessionId) {
  const session = sessions.get(sessionId);
  if (!session) return;

  session.reconnectAttempts++;
  const delay = Math.min(
    1000 * Math.pow(2, session.reconnectAttempts),
    30000
  );

  console.log(`[${sessionId}] Reconnecting in ${delay}ms (attempt ${session.reconnectAttempts})`);
  
  setTimeout(() => {
    if (sessions.has(sessionId)) {
      session.deepgramWs = createDeepgramConnection(sessionId);
    }
  }, delay);
}

// PlayHT TTS streaming with buffer management
async function streamTTSResponse(sessionId, text) {
  const session = sessions.get(sessionId);
  if (!session) return;

  // Cancel any ongoing playback (barge-in handling)
  session.audioBuffer.clear();

  try {
    const response = await fetch('https://api.play.ht/api/v2/tts/stream', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.PLAYHT_API_KEY}`,
        'X-User-ID': process.env.PLAYHT_USER_ID,
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        text: text,
        voice: playhtConfig.voice,
        output_format: playhtConfig.output_format,
        quality: playhtConfig.quality,
        speed: playhtConfig.speed
      })
    });

    if (!response.ok) {
      throw new Error(`PlayHT API error: ${response.status}`);
    }

    const reader = response.body.getReader();
    let audioChunk;

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      // Buffer audio to prevent jitter
      session.audioBuffer.add({
        data: value,
        duration: (value.length / 16000) * 1000 // Estimate duration
      });

      // Send to client WebSocket
      if (session.ws.readyState === WebSocket.OPEN) {
        session.ws.send(value, { binary: true });
      }
    }
  } catch (error) {
    console.error(`[${sessionId}] TTS streaming error:`, error);
    throw error;
  }
}

// Simplified LLM response (replace with your LLM)
async function generateResponse(transcript) {
  // This is where you'd call OpenAI, Anthropic, etc.
  return `You said: ${transcript}. This is a test response.`;
}

// WebSocket connection handler
wss.on('connection', (ws) => {
  const sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
  console.log(`[${sessionId}] Client connected`);

  // Initialize session
  const session = {
    ws: ws,
    deepgramWs: createDeepgramConnection(sessionId),
    audioBuffer: new AudioBuffer(),
    isProcessing: false,
    reconnectAttempts: 0,
    createdAt: Date.now()
  };
  sessions.set(sessionId, session);

  // Auto-cleanup after TTL
  setTimeout(() => {
    if (sessions.has(sessionId)) {
      console.log(`[${sessionId}] Session expired (TTL)`);
      cleanupSession(sessionId);
    }
  }, SESSION_TTL);

  ws.on

## FAQ

## Technical Questions

**How do I handle WebSocket reconnection when Deepgram drops mid-stream?**

Implement exponential backoff with a maximum delay cap. Track `reconnectAttempts` and increment after each failed connection. Set `maxReconnectDelay` to 30 seconds to prevent runaway retry loops. When the WebSocket closes unexpectedly, calculate `delay = Math.min(1000 * Math.pow(2, reconnectAttempts), maxReconnectDelay)`, then reconnect. Store partial transcripts in memory before reconnecting so you don't lose context. Most production failures happen because developers retry immediately without backoff—this will exhaust your connection pool.

**What's the latency difference between batch STT and streaming STT?**

Batch processing (send entire audio file) adds 500ms–2s overhead for queueing and processing. Streaming STT with Deepgram returns partial transcripts within 100–300ms of audio arrival, depending on network jitter and VAD (voice activity detection) settings. For real-time voice stacks, streaming is non-negotiable. Batch is only acceptable for post-call analysis or asynchronous transcription jobs.

**How do I prevent PlayHT from speaking over Deepgram's STT?**

Set `isProcessing = true` when audio input starts, and only allow TTS output when `isProcessing = false`. Use a state machine, not boolean flags alone—this prevents race conditions where both streams try to output simultaneously. If the user interrupts mid-sentence, flush the `playhtStream` buffer immediately and set `interrupted = true` to signal cancellation downstream.

**Why does my voice stack have 2–3 second latency spikes?**

Check three things: (1) LLM response time (usually 500–1500ms for GPT-4), (2) TTS synthesis delay (PlayHT typically 200–800ms depending on text length), (3) network jitter on WebSocket. Profile each component separately using `console.time()`. Most developers blame the API when the bottleneck is their own LLM integration.

## Performance

**What sample rate should I use for Deepgram?**

Use 16kHz for voice conversations (standard for telephony). 8kHz works but reduces accuracy by 3–5%. Higher rates (48kHz) waste bandwidth without meaningful accuracy gains for speech. Set `sample_rate: 16000` in `deepgramConfig` and ensure your `mediaRecorder` or audio source matches this rate. Mismatched sample rates cause audio artifacts and transcription errors.

**How do I reduce PlayHT synthesis latency?**

Lower the `quality` setting from "high" to "medium" (saves 200–400ms). Reduce `speed` to 0.9–1.0 (faster speech = less synthesis time, but clarity suffers). Pre-warm the connection by sending a test request during initialization. Batch multiple short sentences into one TTS call instead of calling PlayHT for every single response chunk—this reduces API overhead by 40–60%.

## Platform Comparison

**Should I use Deepgram or Google Cloud Speech-to-Text?**

Deepgram is 2–3x faster for streaming (100–200ms latency vs. 300–500ms for Google). Deepgram's pricing is predictable (per-minute). Google charges per request plus storage. For real-time voice stacks, Deepgram wins on latency and cost. Google wins if you need multi-language support across 100+ languages out-of-the-box.

**PlayHT vs. ElevenLabs for TTS?**

PlayHT has lower latency (200–600ms) and better cost efficiency for high-volume applications. ElevenLabs has superior voice quality and emotional expressiveness, but adds 400–1000ms latency. For conversational AI, PlayHT is the practical choice. For branded voice experiences, ElevenLabs justifies the latency trade-off.

## Resources

**Deepgram Speech-to-Text API**
- [Official Documentation](https://developers.deepgram.com/docs) – Real-time STT, WebSocket streaming, VAD configuration
- [API Reference](https://developers.deepgram.com/reference) – Endpoint specs, authentication, error codes

**PlayHT Text-to-Speech API**
- [Official Documentation](https://docs.playht.com) – Low-latency TTS streaming, voice selection, output formats
- [API Reference](https://docs.playht.com/api-reference) – Endpoint specs, authentication, streaming protocols

**Voice AI Stack Architecture**
- [Deepgram GitHub Examples](https://github.com/deepgram-devs) – Production WebSocket implementations, audio streaming patterns
- [PlayHT GitHub Examples](https://github.com/playht) – End-to-end conversational AI pipeline examples