API Documentation

# Check Docker startup logs for the admin token:
docker logs whissle-gateway 2>&1 | grep "Token:"

# Or read it from the persistent data volume:
docker exec whissle-gateway cat /data/auth/admin_token.txt

# Create a token for an application or user
curl -X POST http://localhost:9000/auth/tokens \
  -H "Authorization: Bearer wh_YOUR_ADMIN_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"user_id": "my-app", "label": "My Application"}'

# Response:
# {"success": true, "token": "wh_a1b2c3...", "user_id": "my-app", ...}

# REST — Authorization header
curl -X POST http://localhost:9000/asr/transcribe \
  -H "Authorization: Bearer wh_YOUR_TOKEN" \
  -F "file=@call.mp3" -F "diarize=true"

# WebSocket — query parameter
wscat -c "ws://localhost:9000/listen?token=wh_YOUR_TOKEN"
wscat -c "ws://localhost:9000/asr/stream?token=wh_YOUR_TOKEN"

curl -X POST http://localhost:8001/transcribe \
    -F "file=@call.mp3"

# With diarization + metadata + AI analysis
curl -X POST http://localhost:8001/transcribe \
    -F "file=@call.mp3" \
    -F "diarize=true" \
    -F "num_speakers=2" \
    -F "punctuation=true" \
    -F "itn=true" \
    -F "metadata_prob=true" \
    -F "word_timestamps=true" \
    -F "speech_analysis=true" \
    -F "summarize=sales_coaching" \
    -o result.json

{
  "transcript": "Hello, good morning. How are you?",
  "duration": 5.2,
  "inference_time": "0.823s",
  "model": "whissle-large",
  "decoder": "beam_search_kenlm",

  // Present when diarize=true
  "num_speakers": 2,
  "segments": [
    {
      "speaker": "SPEAKER_00",
      "text": "Hello, good morning.",
      "start": 1.0,
      "end": 1.9,
      "confidence": 0.96,

      // Always present — top-1 prediction per category
      "metadata": {
        "emotion": "EMOTION_NEUTRAL",
        "behavior": "BEHAVIOR_DIRECT",
        "role": "ROLE_INTERVIEWER",
        "eval": "EVAL_NONE",
        "age": "AGE_30_45",
        "gender": "GENDER_MALE"
      },

      // Present when metadata_prob=true — full distributions
      "metadata_probs": {
        "emotion": {
          "EMOTION_NEUTRAL": 0.82,
          "EMOTION_HAPPY": 0.12,
          "EMOTION_SAD": 0.03,
          "EMOTION_ANGRY": 0.02,
          "EMOTION_FEAR": 0.01
        },
        "age": { "AGE_30_45": 0.65, "AGE_18_30": 0.25, ... }
      },

      // Present when word_timestamps=true
      "words": [
        {"word": "Hello", "start": 1.0, "end": 1.3, "confidence": 0.98},
        {"word": "good",  "start": 1.4, "end": 1.6, "confidence": 0.95},
        {"word": "morning", "start": 1.6, "end": 1.9, "confidence": 0.97}
      ],

      "entities": [
        {"type": "PERSON", "value": "Diana"},
        {"type": "ORG", "value": "American Amicable"}
      ]
    }
  ],

  // Present when speech_analysis=true
  "speech_analysis": {
    "lexical_fluency": 0.92,
    "vocabulary_range": 0.85,
    "grammar_score": 0.88
  },

  // Present when summarize=... is set
  "summary": "...",           // text or markdown
  "analysis": { ... }         // structured JSON (sales_coaching, collections)
}

WebSocket ws://localhost:8001/stream

# Protocol:
# 1. Connect
# 2. Send JSON config
# 3. Send binary PCM chunks (16kHz, 16-bit, mono)
# 4. Receive JSON transcripts (interim + final)
# 5. Send {"type": "end"} to finalize

import asyncio, json, websockets

async def stream(audio_path):
    async with websockets.connect("ws://localhost:8001/stream") as ws:
        # 1. Config
        await ws.send(json.dumps({
            "type": "config",
            "sample_rate": 16000,
            "language": "en",
            "model": "en-in-tech-misc",  # optional
            "interim_results": True
        }))

        # 2. Stream audio chunks (100ms = 3200 bytes at 16kHz s16le)
        with open(audio_path, "rb") as f:
            while chunk := f.read(3200):
                await ws.send(chunk)
                await asyncio.sleep(0.05)  # pace to real-time

        # 3. Signal end
        await ws.send(json.dumps({"type": "end"}))

        # 4. Receive results
        async for msg in ws:
            data = json.loads(msg)
            if data.get("is_final"):
                print(f"FINAL: {data['text']}")
                print(f"  emotion={data['metadata']['emotion']}")
            elif data["type"] == "transcript":
                print(f"  interim: {data['text']}")
            elif data["type"] == "end":
                break

asyncio.run(stream("call.wav"))

// Interim transcript (partial, updates as more audio arrives)
{
  "type": "transcript",
  "channel": "microphone",
  "text": "Hello how are",
  "is_final": false,
  "utterance_end": false
}

// Final transcript (complete utterance with metadata)
{
  "type": "transcript",
  "text": "Hello, how are you?",
  "is_final": true,
  "utterance_end": true,
  "process_ms": 510,
  "metadata": {
    "emotion": "EMOTION_NEUTRAL",
    "behavior": "BEHAVIOR_DIRECT",
    "role": "ROLE_INTERVIEWER",
    "age": "AGE_30_45",
    "gender": "GENDER_MALE"
  }
}

// Session end
{ "type": "end" }

# Batch with diarization
curl -X POST http://localhost:8001/transcribe \
    -F "file=@meeting.wav" \
    -F "diarize=true" \
    -F "num_speakers=3"

# Response includes per-speaker segments:
# [SPEAKER_00]: "Hello, good morning."
# [SPEAKER_01]: "Hi, how are you?"
# [SPEAKER_00]: "I'm doing well, thank you."
# [SPEAKER_02]: "Let's get started."

# Each segment has its own metadata (emotion, age, gender per speaker)

curl -F "file=@call.mp3" -F "diarize=true" \
    -F "summarize=sales_coaching" http://localhost:8001/transcribe

# Returns analysis.overall_score, analysis.practices,
# analysis.buyer_outcome, analysis.highlights,
# analysis.behaviors (per-segment labels)

-F "summarize=collections"

# Returns analysis.compliance, analysis.call_outcome,
# analysis.customer_sentiment, analysis.next_action

-F "summarize=true"

-F "summarize=You are a medical call quality analyst. \
    Score this call on: 1) empathy (0-10), \
    2) medical history completeness, 3) HIPAA compliance. \
    Return JSON with scores and recommendations."

# The LLM receives your prompt + every segment with
# emotion, intent, behavior, age, gender, role labels.

# Push a frame
curl -X POST http://localhost:8002/context/session-123 \
    -F "image=<base64-encoded-jpeg>"

# Get current visual context
curl http://localhost:8002/context/session-123

# Response:
{
  "fast": {
    "faces": [{
      "emotion": "happy",
      "gaze": "center",
      "head_pose": {"pitch": -2.1, "yaw": 5.3, "roll": 0.8},
      "speaking": true
    }],
    "hands": [{"gesture": "open_palm", "confidence": 0.92}]
  },
  "semantic": {
    "scene": "modern office with whiteboard",
    "activity": "presenting to camera",
    "entities": ["laptop", "whiteboard", "coffee mug"]
  },
  "prompt_block": "[VISUAL CONTEXT]\nFace: happy, looking at camera..."
}

# Video-only streaming
wscat -c "ws://localhost:8002/ws?session_id=s1"
# Send: {"type":"frame","image":"<b64 jpeg>"}
# Receive: {"type":"fast","faces":[...],"hands":[...]}
# Receive: {"type":"semantic","scene":"...","activity":"..."}

# Fused audio-visual streaming (single WebSocket)
# Binary messages → forwarded to ASR as PCM audio
# JSON messages  → processed through vision lanes
# ASR transcripts are enriched with latest visual context
# before forwarding back to the client

curl -X POST http://localhost:8002/analyze \
    -F "file=@interview.mp4" \
    -F "fast_fps=5" \
    -F "semantic_fps=0.5" \
    -F "asr=true"

# Returns timeline of visual context + ASR transcript
# with fused audio-visual results per segment

WebSocket ws://localhost:8003/stream

# Protocol:
# 1. Connect
# 2. Send config: {"type": "config", "voice": "af_heart"}
# 3. Send speak:  {"type": "speak", "text": "Hello!"}
# 4. Receive binary PCM s16le chunks (24kHz mono)
# 5. Receive {"type": "done"} when complete

import asyncio, json, wave, websockets

async def speak(text, voice="af_heart"):
    async with websockets.connect("ws://localhost:8003/stream") as ws:
        await ws.send(json.dumps({"type": "config", "voice": voice}))
        await ws.send(json.dumps({"type": "speak", "text": text}))

        audio = b""
        async for msg in ws:
            if isinstance(msg, bytes):
                audio += msg           # PCM s16le, 24kHz
            elif json.loads(msg).get("type") == "done":
                break

        with wave.open("out.wav", "wb") as wf:
            wf.setnchannels(1)
            wf.setsampwidth(2)
            wf.setframerate(24000)
            wf.writeframes(audio)

asyncio.run(speak("Hello from Whissle Gateway."))

curl http://localhost:8003/voices

# Returns: {"voices": {"af_heart": {...}, "am_adam": {...}, ...}}

# Create a voice agent
curl -X POST http://localhost:9000/bot/api/agents \
    -H "Content-Type: application/json" \
    -b "access_token=..." \
    -d '{
      "name": "Sales Rep",
      "system_prompt": "You are a helpful sales assistant for Acme Corp.",
      "greeting": "Hi there! How can I help you today?",
      "voice": "tara",
      "voice_gender": "female",
      "llm_model": "gemini-2.5-flash",
      "video_enabled": true
    }'

# Response:
# {"id": "uuid", "name": "Sales Rep", "voice": "tara", ...}
# video_enabled: when true, camera is enabled by default for this agent.
# Can also be overridden per-session with ?video=1 on the offer endpoint.

# Start a WebRTC voice session
curl -X POST http://localhost:9000/bot/api/offer \
    -H "Content-Type: application/json" \
    -b "access_token=..." \
    -d '{
      "sdp": "<browser SDP offer string>",
      "type": "offer",
      "agent_id": "agent-uuid",
      "customer_id": "customer-uuid"
    }'

# Response: {"sdp": "<SDP answer>", "type": "answer", "pc_id": "..."}

# Start with camera enabled (video intelligence)
curl -X POST "http://localhost:9000/bot/api/offer?video=1" \
    -H "Content-Type: application/json" \
    -b "access_token=..." \
    -d '{"sdp": "...", "type": "offer", "agent_id": "..."}'

# Start with 3D avatar
curl -X POST "http://localhost:9000/bot/api/offer?avatar=simli\
    &avatar_face_id=b9e5fba3-..." \
    -d '{"sdp": "...", "type": "offer", "agent_id": "..."}'

# Send ICE candidates as they arrive
curl -X PATCH http://localhost:9000/bot/api/offer \
    -H "Content-Type: application/json" \
    -d '{"pc_id": "...", "candidates": [{"candidate": "..."}]}'

curl -X POST http://localhost:9000/bot/api/customers \
    -H "Content-Type: application/json" \
    -b "access_token=..." \
    -d '{
      "name": "Jane Smith",
      "phone_number": "+14155551234",
      "email": "jane@example.com"
    }'

transport.input (WebRTC / Twilio audio + video)
    → VideoContext       Captures video frames → POST to :8002 (~2.5 fps)
    → WhissleSTT         ASR via local :8001, returns text + metadata
    → SmartEndpointer    Adapts VAD from intent head (open question = longer wait)
    → EmotionAdaptive    Injects emotion/intent + visual context into LLM
    → Backchannel        Plays "mm-hmm" cues during user speech
    → UserAggregator     Combines multi-utterance user turns
    → LLM               Generates response with speech + visual context
    → ClosingPhrase      Detects goodbye → auto-hangup
    → NumberPronouncer   "4500" → "forty-five hundred"
    → TTS (Kokoro)       Text-to-speech via local :8003
    → AudioHumanizer     Adds room tone + breath sounds
    → SilenceWatchdog    2-strike: prompt after silence, hangup after second
    → transport.output   WebRTC / Twilio audio out
    → AudioBuffer        Records both directions for call recording
    → AssistantAgg       Combines multi-chunk assistant turns

POST http://localhost:8765/process

curl -X POST http://localhost:8765/process \
    -H "Content-Type: application/json" \
    -H "X-Device-Id: user-1" \
    -d '{
      "transcript": "What meetings do I have today?",
      "user_id": "user-1",
      "language": "en"
    }'

# Response: mode, processed_text, is_command, etc.

POST http://localhost:8765/voice-agent/chat/stream

curl -N -X POST http://localhost:8765/voice-agent/chat/stream \
    -H "Content-Type: application/json" \
    -d '{
      "messages": [
        {"role": "user", "content": "Summarize the key takeaways."}
      ],
      "user_id": "user-1"
    }'

# SSE response:
# data: {"type": "text_chunk", "content": "The key takeaways..."}
# data: {"type": "done"}

# Sign up
curl -X POST http://localhost:9000/bot/api/auth/signup \
    -H "Content-Type: application/json" \
    -d '{"email": "user@example.com", "password": "securepass123"}'

# Login — response sets HTTP-only cookies
curl -X POST http://localhost:9000/bot/api/auth/login \
    -H "Content-Type: application/json" \
    -c cookies.txt \
    -d '{"email": "user@example.com", "password": "securepass123"}'

# Use cookies for authenticated requests
curl http://localhost:9000/bot/api/auth/me -b cookies.txt

# Guest access (no credentials needed)
curl -X POST http://localhost:9000/bot/api/auth/guest -c cookies.txt

# Create an organization
curl -X POST http://localhost:9000/bot/api/orgs \
    -H "Content-Type: application/json" \
    -b cookies.txt \
    -d '{"name": "Acme Corp", "slug": "acme"}'

# Switch active organization
curl -X POST http://localhost:9000/bot/api/orgs/switch \
    -H "Content-Type: application/json" \
    -b cookies.txt \
    -d '{"organization_id": "org-uuid"}'

# Invite a user
curl -X POST http://localhost:9000/bot/api/orgs/{org_id}/invitations \
    -H "Content-Type: application/json" \
    -b cookies.txt \
    -d '{"email": "colleague@example.com", "role": "member"}'

# Accept invitation (the invited user)
curl -X POST http://localhost:9000/bot/api/invitations/accept \
    -H "Content-Type: application/json" \
    -b cookies.txt \
    -d '{"token": "invitation-token-from-email"}'