Live conversation

// Node 18+ with Express. Two routes the browser calls.
import express from "express";

const app = express();
const KEY = process.env.EESI_API_KEY;
const API = "https://api.eesi.ai/v1";

app.get("/nur/ice", async (_req, res) => {
  const r = await fetch(`${API}/realtime/ice-servers`, {
    headers: { Authorization: `Bearer ${KEY}` },
  });
  res.json(await r.json());
});

app.post("/nur/call", express.text({ type: "application/sdp" }), async (req, res) => {
  const r = await fetch(`${API}/realtime/calls?model=nur-v1`, {
    method: "POST",
    headers: { Authorization: `Bearer ${KEY}`, "Content-Type": "application/sdp" },
    body: req.body,
  });
  res.status(r.status).type("application/sdp").send(await r.text());
});

app.listen(3000);

const { ice_servers } = await fetch("/nur/ice").then((r) => r.json());
const pc = new RTCPeerConnection({ iceServers: ice_servers });

// Nur's voice
const speaker = new Audio();
speaker.autoplay = true;
pc.ontrack = (event) => { speaker.srcObject = event.streams[0]; };

// Your voice
const mic = await navigator.mediaDevices.getUserMedia({
  audio: { echoCancellation: true, noiseSuppression: true, autoGainControl: true },
});
pc.addTrack(mic.getAudioTracks()[0], mic);

// Events travel on a data channel with this exact name
const events = pc.createDataChannel("oai-events");
events.onopen = () => {
  events.send(JSON.stringify({
    type: "session.update",
    session: {
      type: "realtime",
      instructions: "You are a friendly concierge. Keep answers short.",
      audio: { output: { voice: "ev_d116d96b" } },
    },
  }));
  events.send(JSON.stringify({ type: "response.create" })); // Nur speaks first
};
events.onmessage = (event) => console.log(JSON.parse(event.data).type);

// Send the offer once candidates are gathered
await pc.setLocalDescription(await pc.createOffer());
await new Promise((resolve) => {
  if (pc.iceGatheringState === "complete") return resolve();
  pc.addEventListener("icegatheringstatechange", () => {
    if (pc.iceGatheringState === "complete") resolve();
  });
  setTimeout(resolve, 2000);
});
const answer = await fetch("/nur/call", {
  method: "POST",
  headers: { "Content-Type": "application/sdp" },
  body: pc.localDescription.sdp,
}).then((r) => r.text());
await pc.setRemoteDescription({ type: "answer", sdp: answer });