Talk to your agent: speech in, speech out

import io
import re

import pyaudio
import sounddevice as sd
import soundfile as sf
from openai import OpenAI

client = OpenAI(api_key="sk-YOUR-FLUX-KEY", base_url="https://api.fluxrouter.ai/v1")
speaker = pyaudio.PyAudio().open(format=pyaudio.paInt16, channels=1, rate=24000, output=True)

RATE = 16000
SENTENCE_END = re.compile(r"(?<=[.!?])\s+")
history = [{"role": "system", "content": "You are a voice assistant. Answer in short, plain sentences."}]


def record(seconds: float = 5.0) -> io.BytesIO:
    input(f"Press Enter, then speak for {seconds:.0f} seconds...")
    audio = sd.rec(int(seconds * RATE), samplerate=RATE, channels=1, dtype="int16")
    sd.wait()
    buf = io.BytesIO()
    sf.write(buf, audio, RATE, format="WAV")
    buf.seek(0)
    buf.name = "question.wav"
    return buf


def speak(sentence: str) -> float:
    with client.audio.speech.with_streaming_response.create(
        model="flux-voice-speak",
        voice="nova",
        input=sentence,
        response_format="pcm",
    ) as response:
        cost = float(response.headers.get("x-flux-cost-usd", 0))
        for chunk in response.iter_bytes(chunk_size=4096):
            speaker.write(chunk)
    return cost


while True:
    # 1. Listen.
    raw = client.audio.transcriptions.with_raw_response.create(model="flux-voice", file=record())
    question = raw.parse().text
    turn_cost = float(raw.headers.get("x-flux-cost-usd", 0))
    print("you:", question)
    history.append({"role": "user", "content": question})

    # 2. Think, streamed. 3. Speak each sentence as soon as it is complete.
    answer, pending = "", ""
    stream = client.chat.completions.create(model="flux-auto", messages=history, stream=True)
    for chunk in stream:
        if not chunk.choices:
            continue
        pending += chunk.choices[0].delta.content or ""
        *done, pending = SENTENCE_END.split(pending)
        for sentence in done:
            answer += sentence + " "
            turn_cost += speak(sentence)
    if pending.strip():
        answer += pending
        turn_cost += speak(pending)

    history.append({"role": "assistant", "content": answer.strip()})
    print("agent:", answer.strip())
    print(f"speech cost this turn: ${turn_cost:.4f} (chat is on your usage page)")

import { spawn } from "node:child_process";
import fs from "node:fs";
import { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import OpenAI from "openai";

const client = new OpenAI({
  apiKey: "sk-YOUR-FLUX-KEY",
  baseURL: "https://api.fluxrouter.ai/v1",
});

const SENTENCE_END = /(?<=[.!?])\s+/;

async function speak(sentence: string): Promise<number> {
  const res = await client.audio.speech.create({
    model: "flux-voice-speak",
    voice: "nova",
    input: sentence,
    response_format: "pcm",
  });
  const player = spawn(
    "ffplay",
    ["-f", "s16le", "-ar", "24000", "-ch_layout", "mono", "-nodisp", "-autoexit", "-loglevel", "quiet", "-"],
    { stdio: ["pipe", "ignore", "ignore"] },
  );
  const done = new Promise((resolve) => player.on("close", resolve));
  await pipeline(Readable.fromWeb(res.body as any), player.stdin);
  await done;
  return Number(res.headers.get("x-flux-cost-usd") ?? 0);
}

// 1. Listen.
const { data: heard, response: raw } = await client.audio.transcriptions
  .create({ model: "flux-voice", file: fs.createReadStream("question.m4a") })
  .withResponse();
let turnCost = Number(raw.headers.get("x-flux-cost-usd") ?? 0);
console.log("you:", heard.text);

// 2. Think, streamed. 3. Speak each sentence as soon as it is complete.
const stream = await client.chat.completions.create({
  model: "flux-auto",
  stream: true,
  messages: [
    { role: "system", content: "You are a voice assistant. Answer in short, plain sentences." },
    { role: "user", content: heard.text },
  ],
});

let pending = "";
for await (const chunk of stream) {
  pending += chunk.choices[0]?.delta?.content ?? "";
  const parts = pending.split(SENTENCE_END);
  pending = parts.pop() ?? "";
  for (const sentence of parts) turnCost += await speak(sentence);
}
if (pending.trim()) turnCost += await speak(pending);

console.log(`speech cost this turn: $${turnCost.toFixed(4)} (chat is on your usage page)`);