WebRTC over LiveKit
We are providing WebRTC via LiveKit as a way to connect to the Flow service. WebRTC is more suitable for mobile applications or for deployments where internet access is bad. More information can be found on official webrtc website.
API
Client is making a HTTP POST request to /v1/flow/livekit endpoint, with body containing StartConversation message as described in the Flow API reference. "audio_format" field should not be used in this case as LiveKit WebRTC takes control of the audio format.
{
"message": "StartConversation",
"conversation_config": {
"template_id": "flow-service-assistant-one",
"template_variables": {
"timezone": "Europe/London"
}
}
}
}
Response
In response our server will create a LiveKit room and return the URL and token needed to connect to the LiveKit server.
{
"url": "wss://test-app-d3kro1gz.livekit.cloud",
"token": "<valid JWT token>",
"id": "<a request id>"
}
Connecting to LiveKit
Provided JWT token has short TTL and should be used immediately after receiving it.
Livekit SDK for a given platform should be used to connect to the LiveKit server. The SDK will handle the connection and audio streaming. Audio is handled by the livekit SDK, so the client does not need to handle audio encoding/decoding. Text messages, or control messages are exchanged using LocalParticipant object. Protocol for exchanged messages is the same as in case of normal WebSocket connection to Flow service.
LiveKit documentation can be found here
Example client in JavaScript
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Minimal LiveKit Flow Client</title>
<style>
body {
font-family: sans-serif;
max-width: 600px;
margin: 0 auto;
padding: 20px;
}
.status {
margin: 10px 0;
padding: 10px;
background-color: #f0f0f0;
}
button {
padding: 10px 20px;
font-size: 16px;
cursor: pointer;
}
#authtoken {
width: 100%;
padding: 8px;
border: 1px solid #ccc;
border-radius: 4px;
font-family: monospace;
box-sizing: border-box;
}
</style>
<script src="https://cdn.jsdelivr.net/npm/livekit-client/dist/livekit-client.umd.min.js"></script>
</head>
<body>
<h1>Minimal LiveKit Flow Client</h1>
<button id="toggle-session">Start Session</button>
<div class="status">
<input id="authtoken" type="text" placeholder="Enter your auth token here">
</div>
<div class="status">Status: <span id="status">disconnected</span></div>
<div id="audio"></div>
<script>
const API_URL = "https://flow.api.speechmatics.com/v1/flow/livekit";
const TEMPLATE_ID = "flow-service-assistant-amelia";
let isSessionRunning = false;
let room;
let transport;
const toggleButton = document.getElementById('toggle-session');
const statusElement = document.getElementById('status');
const sessionIdElement = document.getElementById('session-id');
toggleButton.addEventListener('click', toggleSession);
class LiveKitTransport {
constructor(room) {
this.room = room;
this.encoder = new TextEncoder();
}
send(data) {
if (this.isOpened()) {
if (typeof data === 'string') {
data = this.encoder.encode(data);
}
this.room.localParticipant.publishData(data, { reliable: true });
} else {
console.error("LiveKit room not connected but trying to send data");
}
}
isOpened() {
return this.room && this.room.state === LivekitClient.ConnectionState.Connected;
}
async close() {
await disconnectLiveKit();
}
}
async function toggleSession() {
if (isSessionRunning) {
stopSession();
} else {
startSession();
}
}
async function startSession() {
try {
console.log("Starting session " + API_URL);
toggleButton.disabled = true;
statusElement.textContent = 'connecting';
let AUTH_TOKEN = document.getElementById('authtoken').value;
// Prepare the request to get LiveKit credentials
const response = await fetch(API_URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
"Authorization": `Bearer ${AUTH_TOKEN}`,
},
body: JSON.stringify({
message: "StartConversation",
conversation_config: {
template_id: TEMPLATE_ID
}
}),
});
if (!response.ok) {
throw new Error(`Request failed: ${response.status} ${response.statusText}`);
}
const data = await response.json();
console.log("LiveKit connection info:", data);
sessionIdElement.textContent = data.session_id;
await connectLiveKit(data);
statusElement.textContent = 'connected';
toggleButton.textContent = 'Stop Session';
isSessionRunning = true;
} catch (error) {
console.error("Failed to start session:", error);
statusElement.textContent = 'error';
alert("Failed to start session: " + error.message);
} finally {
toggleButton.disabled = false;
}
}
function stopSession() {
toggleButton.disabled = true;
if (transport && transport.isOpened()) {
transport.send(JSON.stringify({
message: "AudioEnded"
}));
// Allow time for the message to be sent
setTimeout(async () => {
await transport.close();
statusElement.textContent = 'disconnected';
toggleButton.textContent = 'Start Session';
isSessionRunning = false;
toggleButton.disabled = false;
}, 1000);
} else {
statusElement.textContent = 'disconnected';
toggleButton.textContent = 'Start Session';
isSessionRunning = false;
toggleButton.disabled = false;
}
}
async function connectLiveKit(livekitInfo) {
room = new LivekitClient.Room({});
transport = new LiveKitTransport(room);
const decoder = new TextDecoder();
room.on(LivekitClient.RoomEvent.TrackSubscribed, (track, publication, participant) => {
console.log("Track subscribed:", track.kind);
const element = track.attach();
const audioContainer = document.getElementById("audio");
while (audioContainer.firstChild) {
audioContainer.removeChild(audioContainer.firstChild);
}
audioContainer.appendChild(element);
});
room.on(LivekitClient.RoomEvent.DataReceived, (payload, participant, kind) => {
try {
const strData = decoder.decode(payload);
const data = JSON.parse(strData);
console.log("Received data:", data);
// Handle specific messages if needed
if (data.message === "ConversationEnded") {
console.log("Conversation ended");
stopSession();
}
} catch (error) {
console.error("Error processing received data:", error);
}
});
await room.connect(livekitInfo.url, livekitInfo.token);
console.log(`Connected to LiveKit room ${room.name}`);
await room.localParticipant.setCameraEnabled(false);
await room.localParticipant.setMicrophoneEnabled(true, {
autoGainControl: false,
noiseSuppression: true,
voiceIsolation: true,
echoCancellation: true,
});
}
async function disconnectLiveKit() {
if (room) {
await room.disconnect();
console.log("Disconnected from LiveKit room");
room = null;
const audioContainer = document.getElementById("audio");
while (audioContainer.firstChild) {
audioContainer.removeChild(audioContainer.firstChild);
}
}
}
window.addEventListener('beforeunload', () => {
if (isSessionRunning) {
disconnectLiveKit();
}
});
</script>
</body>
</html>