EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"
MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import tempfile
import uuid
from enum import Enum
from shutil import which
from typing import Literal

import edge_tts
import whisper
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds
WHISPER_MODEL_SIZE = "base"
EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Whisper model (lazy singleton) ─────────────────────────────────

_whisper_model: whisper.Whisper | None = None

def _get_whisper_model() -> whisper.Whisper:
    """Return the Whisper model, loading it on first call."""
    global _whisper_model
    if _whisper_model is None:
        logger.info("Loading Whisper '%s' model…", WHISPER_MODEL_SIZE)
        _whisper_model = whisper.load_model(WHISPER_MODEL_SIZE)
    return _whisper_model

# ── STT / TTS pipeline ────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text via Whisper.

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    model = _get_whisper_model()
    with tempfile.NamedTemporaryFile(suffix=".opus", delete=True) as f:
        f.write(audio_bytes)
        f.flush()
        result = await asyncio.to_thread(
            model.transcribe, f.name, language="zh"
        )
    text = result["text"].strip()
    logger.info("STT: transcribed → %s", text)
    return text

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio via edge-tts and ffmpeg.

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    Raises:
        RuntimeError: If ffmpeg conversion fails.
    """
    logger.info("TTS: synthesising → %s", text)

    # 1. edge-tts → MP3
    communicate = edge_tts.Communicate(text=text, voice=EDGE_TTS_VOICE)
    mp3_chunks: list[bytes] = []
    async for chunk in communicate.stream():
        if chunk["type"] == "audio":
            mp3_chunks.append(chunk["data"])
    mp3_bytes = b"".join(mp3_chunks)

    # 2. MP3 → Opus via ffmpeg
    proc = await asyncio.create_subprocess_exec(
        FFMPEG_BIN, "-i", "pipe:0",
        "-c:a", "libopus", "-b:a", "24k",
        "-ar", "16000", "-ac", "1",
        "-f", "opus", "pipe:1",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    opus_bytes, stderr = await proc.communicate(input=mp3_bytes)

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("ffmpeg failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"ffmpeg TTS conversion error: {err_msg}")

    logger.info("TTS: generated %d bytes of Opus audio", len(opus_bytes))
    return opus_bytes

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

+MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB


Now add `SessionState` after the existing Pydantic models (after `TextReply`):


Edit

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class SessionState(BaseModel):
    """Per-connection state machine for press-to-talk buffering."""

    model_config = {"arbitrary_types_allowed": True}

    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []

    def buffer_size(self) -> int:
        """Return total buffered audio size in bytes."""
        return sum(len(c) for c in self.audio_chunks)

    def drain_audio(self) -> bytes:
        """Join and clear all buffered audio chunks."""
        audio = b"".join(self.audio_chunks)
        self.audio_chunks.clear()
        return audio

    def reset(self) -> None:
        """Reset state to idle with empty buffer."""
        self.status = "idle"
        self.audio_chunks.clear()


The file /root/projects/openclaw-proxy/main.py has been updated successfully.

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class SessionState(BaseModel):
    """Per-connection state machine for press-to-talk buffering."""

    model_config = {"arbitrary_types_allowed": True}

    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []

    def buffer_size(self) -> int:
        """Return total buffered audio size in bytes."""
        return sum(len(c) for c in self.audio_chunks)

    def drain_audio(self) -> bytes:
        """Join and clear all buffered audio chunks."""
        audio = b"".join(self.audio_chunks)
        self.audio_chunks.clear()
        return audio

    def reset(self) -> None:
        """Reset state to idle with empty buffer."""
        self.status = "idle"
        self.audio_chunks.clear()

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import tempfile
import uuid
from enum import Enum
from shutil import which
from typing import Literal

import edge_tts
import whisper
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds
WHISPER_MODEL_SIZE = "base"
EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"
MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Whisper model (lazy singleton) ─────────────────────────────────

_whisper_model: whisper.Whisper | None = None

def _get_whisper_model() -> whisper.Whisper:
    """Return the Whisper model, loading it on first call."""
    global _whisper_model
    if _whisper_model is None:
        logger.info("Loading Whisper '%s' model…", WHISPER_MODEL_SIZE)
        _whisper_model = whisper.load_model(WHISPER_MODEL_SIZE)
    return _whisper_model

# ── STT / TTS pipeline ────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text via Whisper.

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    model = _get_whisper_model()
    with tempfile.NamedTemporaryFile(suffix=".opus", delete=True) as f:
        f.write(audio_bytes)
        f.flush()
        result = await asyncio.to_thread(
            model.transcribe, f.name, language="zh"
        )
    text = result["text"].strip()
    logger.info("STT: transcribed → %s", text)
    return text

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio via edge-tts and ffmpeg.

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    Raises:
        RuntimeError: If ffmpeg conversion fails.
    """
    logger.info("TTS: synthesising → %s", text)

    # 1. edge-tts → MP3
    communicate = edge_tts.Communicate(text=text, voice=EDGE_TTS_VOICE)
    mp3_chunks: list[bytes] = []
    async for chunk in communicate.stream():
        if chunk["type"] == "audio":
            mp3_chunks.append(chunk["data"])
    mp3_bytes = b"".join(mp3_chunks)

    # 2. MP3 → Opus via ffmpeg
    proc = await asyncio.create_subprocess_exec(
        FFMPEG_BIN, "-i", "pipe:0",
        "-c:a", "libopus", "-b:a", "24k",
        "-ar", "16000", "-ac", "1",
        "-f", "opus", "pipe:1",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    opus_bytes, stderr = await proc.communicate(input=mp3_bytes)

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("ffmpeg failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"ffmpeg TTS conversion error: {err_msg}")

    logger.info("TTS: generated %d bytes of Opus audio", len(opus_bytes))
    return opus_bytes

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

+    """Per-connection state machine for press-to-talk buffering."""

+    model_config = {"arbitrary_types_allowed": True}

+    status: Literal["idle", "listening", "processing"] = "idle"

+        """Return total buffered audio size in bytes."""

+        return sum(len(c) for c in self.audio_chunks)

+        """Join and clear all buffered audio chunks."""

+        audio = b"".join(self.audio_chunks)

+        """Reset state to idle with empty buffer."""


Now refactor `handle_text_frame`, `handle_binary_frame`, and add `handle_pipeline`:


async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_text_frame(
    ws: WebSocket, raw: str, state: SessionState, session_id: str,
) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action in (HardwareAction.KEY_DOWN, HardwareAction.WAKE_WORD):
        if state.status == "listening":
            logger.info("Re-key_down while listening — resetting buffer")
            state.audio_chunks.clear()
        state.status = "listening"
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

    elif event.action == HardwareAction.KEY_UP:
        if state.status != "listening":
            logger.warning("key_up in '%s' state — ignored", state.status)
            return
        state.status = "processing"
        await handle_pipeline(ws, state, session_id)


async def handle_text_frame(
    ws: WebSocket, raw: str, state: SessionState, session_id: str,
) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action in (HardwareAction.KEY_DOWN, HardwareAction.WAKE_WORD):
        if state.status == "listening":
            logger.info("Re-key_down while listening — resetting buffer")
            state.audio_chunks.clear()
        state.status = "listening"
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

    elif event.action == HardwareAction.KEY_UP:
        if state.status != "listening":
            logger.warning("key_up in '%s' state — ignored", state.status)
            return
        state.status = "processing"
        await handle_pipeline(ws, state, session_id)

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import tempfile
import uuid
from enum import Enum
from shutil import which
from typing import Literal

import edge_tts
import whisper
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds
WHISPER_MODEL_SIZE = "base"
EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"
MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class SessionState(BaseModel):
    """Per-connection state machine for press-to-talk buffering."""

    model_config = {"arbitrary_types_allowed": True}

    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []

    def buffer_size(self) -> int:
        """Return total buffered audio size in bytes."""
        return sum(len(c) for c in self.audio_chunks)

    def drain_audio(self) -> bytes:
        """Join and clear all buffered audio chunks."""
        audio = b"".join(self.audio_chunks)
        self.audio_chunks.clear()
        return audio

    def reset(self) -> None:
        """Reset state to idle with empty buffer."""
        self.status = "idle"
        self.audio_chunks.clear()

# ── Whisper model (lazy singleton) ─────────────────────────────────

_whisper_model: whisper.Whisper | None = None

def _get_whisper_model() -> whisper.Whisper:
    """Return the Whisper model, loading it on first call."""
    global _whisper_model
    if _whisper_model is None:
        logger.info("Loading Whisper '%s' model…", WHISPER_MODEL_SIZE)
        _whisper_model = whisper.load_model(WHISPER_MODEL_SIZE)
    return _whisper_model

# ── STT / TTS pipeline ────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text via Whisper.

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    model = _get_whisper_model()
    with tempfile.NamedTemporaryFile(suffix=".opus", delete=True) as f:
        f.write(audio_bytes)
        f.flush()
        result = await asyncio.to_thread(
            model.transcribe, f.name, language="zh"
        )
    text = result["text"].strip()
    logger.info("STT: transcribed → %s", text)
    return text

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio via edge-tts and ffmpeg.

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    Raises:
        RuntimeError: If ffmpeg conversion fails.
    """
    logger.info("TTS: synthesising → %s", text)

    # 1. edge-tts → MP3
    communicate = edge_tts.Communicate(text=text, voice=EDGE_TTS_VOICE)
    mp3_chunks: list[bytes] = []
    async for chunk in communicate.stream():
        if chunk["type"] == "audio":
            mp3_chunks.append(chunk["data"])
    mp3_bytes = b"".join(mp3_chunks)

    # 2. MP3 → Opus via ffmpeg
    proc = await asyncio.create_subprocess_exec(
        FFMPEG_BIN, "-i", "pipe:0",
        "-c:a", "libopus", "-b:a", "24k",
        "-ar", "16000", "-ac", "1",
        "-f", "opus", "pipe:1",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    opus_bytes, stderr = await proc.communicate(input=mp3_bytes)

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("ffmpeg failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"ffmpeg TTS conversion error: {err_msg}")

    logger.info("TTS: generated %d bytes of Opus audio", len(opus_bytes))
    return opus_bytes

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-async def handle_text_frame(ws: WebSocket, raw: str) -> None:

+    ws: WebSocket, raw: str, state: SessionState, session_id: str,

+        state: Per-connection session state.

+        session_id: Per-connection session ID.

-    if event.action == HardwareAction.WAKE_WORD:

+    if event.action in (HardwareAction.KEY_DOWN, HardwareAction.WAKE_WORD):

+        if state.status == "listening":

+            logger.info("Re-key_down while listening — resetting buffer")

+    elif event.action == HardwareAction.KEY_UP:

+        if state.status != "listening":

+            logger.warning("key_up in '%s' state — ignored", state.status)

+        await handle_pipeline(ws, state, session_id)


Now replace `handle_binary_frame` and add `handle_pipeline`:


async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

async def handle_binary_frame(ws: WebSocket, data: bytes, state: SessionState) -> None:
    """Buffer an audio binary frame if in listening state.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        state: Per-connection session state.
    """
    if state.status != "listening":
        logger.warning("Binary frame in '%s' state — dropped", state.status)
        return

    if state.buffer_size() + len(data) > MAX_AUDIO_BUFFER_BYTES:
        logger.warning("Audio buffer full (%d bytes) — dropping frame", state.buffer_size())
        return

    state.audio_chunks.append(data)
    logger.info("Buffered %d bytes (total: %d)", len(data), state.buffer_size())

async def handle_pipeline(
    ws: WebSocket, state: SessionState, session_id: str,
) -> None:
    """Drain buffered audio and run STT → LLM → TTS pipeline.

    Args:
        ws: Active WebSocket connection.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    audio = state.drain_audio()
    if not audio:
        logger.warning("key_up with empty buffer — skipping pipeline")
        state.reset()
        return

    try:
        # Stage 1 — STT
        text = await process_stt(audio)

        # Notify hardware: entering thinking phase
        await ws.send_text(StateNotification(status="thinking").model_dump_json())

        # Stage 2 — LLM (with periodic thinking heartbeats)
        async def heartbeat() -> None:
            """Send thinking heartbeat every 3 seconds while LLM is working."""
            while True:
                await asyncio.sleep(3)
                try:
                    await ws.send_text(
                        StateNotification(status="thinking").model_dump_json()
                    )
                except Exception:
                    break

        heartbeat_task = asyncio.create_task(heartbeat())
        try:
            reply = await call_openclaw(text, session_id)
        finally:
            heartbeat_task.cancel()

        await ws.send_text(TextReply(text=reply).model_dump_json())

        # Stage 3 — TTS
        tts_audio = await process_tts(reply)
        logger.info("Pushing %d bytes of audio to client", len(tts_audio))
        await ws.send_bytes(tts_audio)
    finally:
        state.reset()


async def handle_binary_frame(ws: WebSocket, data: bytes, state: SessionState) -> None:
    """Buffer an audio binary frame if in listening state.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        state: Per-connection session state.
    """
    if state.status != "listening":
        logger.warning("Binary frame in '%s' state — dropped", state.status)
        return

    if state.buffer_size() + len(data) > MAX_AUDIO_BUFFER_BYTES:
        logger.warning("Audio buffer full (%d bytes) — dropping frame", state.buffer_size())
        return

    state.audio_chunks.append(data)
    logger.info("Buffered %d bytes (total: %d)", len(data), state.buffer_size())

async def handle_pipeline(
    ws: WebSocket, state: SessionState, session_id: str,
) -> None:
    """Drain buffered audio and run STT → LLM → TTS pipeline.

    Args:
        ws: Active WebSocket connection.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    audio = state.drain_audio()
    if not audio:
        logger.warning("key_up with empty buffer — skipping pipeline")
        state.reset()
        return

    try:
        # Stage 1 — STT
        text = await process_stt(audio)

        # Notify hardware: entering thinking phase
        await ws.send_text(StateNotification(status="thinking").model_dump_json())

        # Stage 2 — LLM (with periodic thinking heartbeats)
        async def heartbeat() -> None:
            """Send thinking heartbeat every 3 seconds while LLM is working."""
            while True:
                await asyncio.sleep(3)
                try:
                    await ws.send_text(
                        StateNotification(status="thinking").model_dump_json()
                    )
                except Exception:
                    break

        heartbeat_task = asyncio.create_task(heartbeat())
        try:
            reply = await call_openclaw(text, session_id)
        finally:
            heartbeat_task.cancel()

        await ws.send_text(TextReply(text=reply).model_dump_json())

        # Stage 3 — TTS
        tts_audio = await process_tts(reply)
        logger.info("Pushing %d bytes of audio to client", len(tts_audio))
        await ws.send_bytes(tts_audio)
    finally:
        state.reset()

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import tempfile
import uuid
from enum import Enum
from shutil import which
from typing import Literal

import edge_tts
import whisper
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds
WHISPER_MODEL_SIZE = "base"
EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"
MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class SessionState(BaseModel):
    """Per-connection state machine for press-to-talk buffering."""

    model_config = {"arbitrary_types_allowed": True}

    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []

    def buffer_size(self) -> int:
        """Return total buffered audio size in bytes."""
        return sum(len(c) for c in self.audio_chunks)

    def drain_audio(self) -> bytes:
        """Join and clear all buffered audio chunks."""
        audio = b"".join(self.audio_chunks)
        self.audio_chunks.clear()
        return audio

    def reset(self) -> None:
        """Reset state to idle with empty buffer."""
        self.status = "idle"
        self.audio_chunks.clear()

# ── Whisper model (lazy singleton) ─────────────────────────────────

_whisper_model: whisper.Whisper | None = None

def _get_whisper_model() -> whisper.Whisper:
    """Return the Whisper model, loading it on first call."""
    global _whisper_model
    if _whisper_model is None:
        logger.info("Loading Whisper '%s' model…", WHISPER_MODEL_SIZE)
        _whisper_model = whisper.load_model(WHISPER_MODEL_SIZE)
    return _whisper_model

# ── STT / TTS pipeline ────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text via Whisper.

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    model = _get_whisper_model()
    with tempfile.NamedTemporaryFile(suffix=".opus", delete=True) as f:
        f.write(audio_bytes)
        f.flush()
        result = await asyncio.to_thread(
            model.transcribe, f.name, language="zh"
        )
    text = result["text"].strip()
    logger.info("STT: transcribed → %s", text)
    return text

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio via edge-tts and ffmpeg.

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    Raises:
        RuntimeError: If ffmpeg conversion fails.
    """
    logger.info("TTS: synthesising → %s", text)

    # 1. edge-tts → MP3
    communicate = edge_tts.Communicate(text=text, voice=EDGE_TTS_VOICE)
    mp3_chunks: list[bytes] = []
    async for chunk in communicate.stream():
        if chunk["type"] == "audio":
            mp3_chunks.append(chunk["data"])
    mp3_bytes = b"".join(mp3_chunks)

    # 2. MP3 → Opus via ffmpeg
    proc = await asyncio.create_subprocess_exec(
        FFMPEG_BIN, "-i", "pipe:0",
        "-c:a", "libopus", "-b:a", "24k",
        "-ar", "16000", "-ac", "1",
        "-f", "opus", "pipe:1",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    opus_bytes, stderr = await proc.communicate(input=mp3_bytes)

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("ffmpeg failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"ffmpeg TTS conversion error: {err_msg}")

    logger.info("TTS: generated %d bytes of Opus audio", len(opus_bytes))
    return opus_bytes

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(
    ws: WebSocket, raw: str, state: SessionState, session_id: str,
) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action in (HardwareAction.KEY_DOWN, HardwareAction.WAKE_WORD):
        if state.status == "listening":
            logger.info("Re-key_down while listening — resetting buffer")
            state.audio_chunks.clear()
        state.status = "listening"
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

    elif event.action == HardwareAction.KEY_UP:
        if state.status != "listening":
            logger.warning("key_up in '%s' state — ignored", state.status)
            return
        state.status = "processing"
        await handle_pipeline(ws, state, session_id)

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:

-    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

+async def handle_binary_frame(ws: WebSocket, data: bytes, state: SessionState) -> None:

+    """Buffer an audio binary frame if in listening state.

-        session_id: Per-connection session ID.

-    Response sequence pushed to client:

-        1. {"type": "state", "status": "thinking"}  — triggers breathing LED

-        2. {"type": "text_reply", "text": "..."}     — agent answer

-        3. <binary>                                   — Opus audio of the answer

+        state: Per-connection session state.

+        logger.warning("Binary frame in '%s' state — dropped", state.status)

-    # Notify hardware: entering thinking phase

-    await ws.send_text(StateNotification(status="thinking").model_dump_json())

+    if state.buffer_size() + len(data) > MAX_AUDIO_BUFFER_BYTES:

+        logger.warning("Audio buffer full (%d bytes) — dropping frame", state.buffer_size())

-    # Stage 2 — LLM (with periodic thinking heartbeats)

-        """Send thinking heartbeat every 3 seconds while LLM is working."""

-                    StateNotification(status="thinking").model_dump_json()

+    logger.info("Buffered %d bytes (total: %d)", len(data), state.buffer_size())

-    heartbeat_task = asyncio.create_task(heartbeat())

+    ws: WebSocket, state: SessionState, session_id: str,

+    """Drain buffered audio and run STT → LLM → TTS pipeline.

+        ws: Active WebSocket connection.

+        session_id: Per-connection session ID.

+        logger.warning("key_up with empty buffer — skipping pipeline")

-        reply = await call_openclaw(text, session_id)

+        text = await process_stt(audio)

-    await ws.send_text(TextReply(text=reply).model_dump_json())

+        # Notify hardware: entering thinking phase

+        await ws.send_text(StateNotification(status="thinking").model_dump_json())

-    logger.info("Pushing %d bytes of audio to client", len(audio))

+        # Stage 2 — LLM (with periodic thinking heartbeats)

+            """Send thinking heartbeat every 3 seconds while LLM is working."""

+                    await ws.send_text(

+                        StateNotification(status="thinking").model_dump_json()

+        heartbeat_task = asyncio.create_task(heartbeat())

+            reply = await call_openclaw(text, session_id)

+        await ws.send_text(TextReply(text=reply).model_dump_json())

+        tts_audio = await process_tts(reply)

+        logger.info("Pushing %d bytes of audio to client", len(tts_audio))


Now update `websocket_chat` to use `SessionState` and pass it to handlers:


    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)

    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    state = SessionState()
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data, state, session_id)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, state)


    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    state = SessionState()
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data, state, session_id)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, state)

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import tempfile
import uuid
from enum import Enum
from shutil import which
from typing import Literal

import edge_tts
import whisper
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds
WHISPER_MODEL_SIZE = "base"
EDGE_TTS_VOICE = "zh-CN-XiaoxiaoNeural"
FFMPEG_BIN = which("ffmpeg") or "ffmpeg"
MAX_AUDIO_BUFFER_BYTES: int = 10 * 1024 * 1024  # 10 MB

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

class SessionState(BaseModel):
    """Per-connection state machine for press-to-talk buffering."""

    model_config = {"arbitrary_types_allowed": True}

    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []

    def buffer_size(self) -> int:
        """Return total buffered audio size in bytes."""
        return sum(len(c) for c in self.audio_chunks)

    def drain_audio(self) -> bytes:
        """Join and clear all buffered audio chunks."""
        audio = b"".join(self.audio_chunks)
        self.audio_chunks.clear()
        return audio

    def reset(self) -> None:
        """Reset state to idle with empty buffer."""
        self.status = "idle"
        self.audio_chunks.clear()

# ── Whisper model (lazy singleton) ─────────────────────────────────

_whisper_model: whisper.Whisper | None = None

def _get_whisper_model() -> whisper.Whisper:
    """Return the Whisper model, loading it on first call."""
    global _whisper_model
    if _whisper_model is None:
        logger.info("Loading Whisper '%s' model…", WHISPER_MODEL_SIZE)
        _whisper_model = whisper.load_model(WHISPER_MODEL_SIZE)
    return _whisper_model

# ── STT / TTS pipeline ────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text via Whisper.

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    model = _get_whisper_model()
    with tempfile.NamedTemporaryFile(suffix=".opus", delete=True) as f:
        f.write(audio_bytes)
        f.flush()
        result = await asyncio.to_thread(
            model.transcribe, f.name, language="zh"
        )
    text = result["text"].strip()
    logger.info("STT: transcribed → %s", text)
    return text

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio via edge-tts and ffmpeg.

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    Raises:
        RuntimeError: If ffmpeg conversion fails.
    """
    logger.info("TTS: synthesising → %s", text)

    # 1. edge-tts → MP3
    communicate = edge_tts.Communicate(text=text, voice=EDGE_TTS_VOICE)
    mp3_chunks: list[bytes] = []
    async for chunk in communicate.stream():
        if chunk["type"] == "audio":
            mp3_chunks.append(chunk["data"])
    mp3_bytes = b"".join(mp3_chunks)

    # 2. MP3 → Opus via ffmpeg
    proc = await asyncio.create_subprocess_exec(
        FFMPEG_BIN, "-i", "pipe:0",
        "-c:a", "libopus", "-b:a", "24k",
        "-ar", "16000", "-ac", "1",
        "-f", "opus", "pipe:1",
        stdin=asyncio.subprocess.PIPE,
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    opus_bytes, stderr = await proc.communicate(input=mp3_bytes)

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("ffmpeg failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"ffmpeg TTS conversion error: {err_msg}")

    logger.info("TTS: generated %d bytes of Opus audio", len(opus_bytes))
    return opus_bytes

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(
    ws: WebSocket, raw: str, state: SessionState, session_id: str,
) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action in (HardwareAction.KEY_DOWN, HardwareAction.WAKE_WORD):
        if state.status == "listening":
            logger.info("Re-key_down while listening — resetting buffer")
            state.audio_chunks.clear()
        state.status = "listening"
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

    elif event.action == HardwareAction.KEY_UP:
        if state.status != "listening":
            logger.warning("key_up in '%s' state — ignored", state.status)
            return
        state.status = "processing"
        await handle_pipeline(ws, state, session_id)

async def handle_binary_frame(ws: WebSocket, data: bytes, state: SessionState) -> None:
    """Buffer an audio binary frame if in listening state.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        state: Per-connection session state.
    """
    if state.status != "listening":
        logger.warning("Binary frame in '%s' state — dropped", state.status)
        return

    if state.buffer_size() + len(data) > MAX_AUDIO_BUFFER_BYTES:
        logger.warning("Audio buffer full (%d bytes) — dropping frame", state.buffer_size())
        return

    state.audio_chunks.append(data)
    logger.info("Buffered %d bytes (total: %d)", len(data), state.buffer_size())

async def handle_pipeline(
    ws: WebSocket, state: SessionState, session_id: str,
) -> None:
    """Drain buffered audio and run STT → LLM → TTS pipeline.

    Args:
        ws: Active WebSocket connection.
        state: Per-connection session state.
        session_id: Per-connection session ID.
    """
    audio = state.drain_audio()
    if not audio:
        logger.warning("key_up with empty buffer — skipping pipeline")
        state.reset()
        return

    try:
        # Stage 1 — STT
        text = await process_stt(audio)

        # Notify hardware: entering thinking phase
        await ws.send_text(StateNotification(status="thinking").model_dump_json())

        # Stage 2 — LLM (with periodic thinking heartbeats)
        async def heartbeat() -> None:
            """Send thinking heartbeat every 3 seconds while LLM is working."""
            while True:
                await asyncio.sleep(3)
                try:
                    await ws.send_text(
                        StateNotification(status="thinking").model_dump_json()
                    )
                except Exception:
                    break

        heartbeat_task = asyncio.create_task(heartbeat())
        try:
            reply = await call_openclaw(text, session_id)
        finally:
            heartbeat_task.cancel()

        await ws.send_text(TextReply(text=reply).model_dump_json())

        # Stage 3 — TTS
        tts_audio = await process_tts(reply)
        logger.info("Pushing %d bytes of audio to client", len(tts_audio))
        await ws.send_bytes(tts_audio)
    finally:
        state.reset()

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-                await handle_text_frame(ws, text_data)

+                await handle_text_frame(ws, text_data, state, session_id)

-                await handle_binary_frame(ws, byte_data, session_id)

+                await handle_binary_frame(ws, byte_data, state)


Updated task #2 status


Updated task #3 status


Now update `test_client.py`:


Write

"""Quick WebSocket client to test the full proxy pipeline."""

import asyncio
import json
import websockets

async def main() -> None:
    uri = "ws://localhost:8000/chat"
    async with websockets.connect(uri) as ws:
        print("=== Connected ===\n")

        # 1) Press-to-talk: key_down
        print("[TX] key_down")
        await ws.send(json.dumps({"type": "hardware_event", "action": "key_down"}))
        resp = json.loads(await ws.recv())
        print(f"[RX] {resp}\n")

        # 2) Send audio frames while "holding button"
        print("[TX] binary audio (80 bytes of mock Opus)")
        await ws.send(b"\x00" * 80)

        # 3) Release button: key_up triggers STT → LLM → TTS pipeline
        print("[TX] key_up")
        await ws.send(json.dumps({"type": "hardware_event", "action": "key_up"}))

        # Collect all responses until we get binary audio back
        while True:
            msg = await ws.recv()
            if isinstance(msg, bytes):
                print(f"[RX] binary audio: {len(msg)} bytes")
                break
            else:
                data = json.loads(msg)
                print(f"[RX] {data}")

        print("\n=== Done ===")

if __name__ == "__main__":
    asyncio.run(main())


The file /root/projects/openclaw-proxy/test_client.py has been updated successfully.

"""Quick WebSocket client to test the full proxy pipeline."""

import asyncio
import json
import websockets

async def main() -> None:
    uri = "ws://localhost:8000/chat"
    async with websockets.connect(uri) as ws:
        print("=== Connected ===\n")

        # 1) Press-to-talk: key_down
        print("[TX] key_down")
        await ws.send(json.dumps({"type": "hardware_event", "action": "key_down"}))
        resp = json.loads(await ws.recv())
        print(f"[RX] {resp}\n")

        # 2) Send audio frames while "holding button"
        print("[TX] binary audio (80 bytes of mock Opus)")
        await ws.send(b"\x00" * 80)

        # 3) Release button: key_up triggers STT → LLM → TTS pipeline
        print("[TX] key_up")
        await ws.send(json.dumps({"type": "hardware_event", "action": "key_up"}))

        # Collect all responses until we get binary audio back
        while True:
            msg = await ws.recv()
            if isinstance(msg, bytes):
                print(f"[RX] binary audio: {len(msg)} bytes")
                break
            else:
                data = json.loads(msg)
                print(f"[RX] {data}")

        print("\n=== Done ===")

if __name__ == "__main__":
    asyncio.run(main())

-        # 1) Test text frame: wake_word

-        await ws.send(json.dumps({"type": "hardware_event", "action": "wake_word"}))

+        await ws.send(json.dumps({"type": "hardware_event", "action": "key_down"}))

-        # 2) Test binary frame: triggers STT → LLM → TTS pipeline

+        # 2) Send audio frames while "holding button"

+        # 3) Release button: key_up triggers STT → LLM → TTS pipeline

+        await ws.send(json.dumps({"type": "hardware_event", "action": "key_up"}))

"""Quick WebSocket client to test the full proxy pipeline."""

import asyncio
import json
import websockets

async def main() -> None:
    uri = "ws://localhost:8000/chat"
    async with websockets.connect(uri) as ws:
        print("=== Connected ===\n")

        # 1) Test text frame: wake_word
        print("[TX] wake_word")
        await ws.send(json.dumps({"type": "hardware_event", "action": "wake_word"}))
        resp = json.loads(await ws.recv())
        print(f"[RX] {resp}\n")

        # 2) Test binary frame: triggers STT → LLM → TTS pipeline
        print("[TX] binary audio (80 bytes of mock Opus)")
        await ws.send(b"\x00" * 80)

        # Collect all responses until we get binary audio back
        while True:
            msg = await ws.recv()
            if isinstance(msg, bytes):
                print(f"[RX] binary audio: {len(msg)} bytes")
                break
            else:
                data = json.loads(msg)
                print(f"[RX] {data}")

        print("\n=== Done ===")

if __name__ == "__main__":
    asyncio.run(main())


Updated task #4 status


Bash

source venv/bin/activate && python -m pytest tests/ -v 2>&1

Run all tests


============================= test session starts ==============================
platform linux -- Python 3.11.6, pytest-9.0.2, pluggy-1.6.0 -- /root/projects/openclaw-proxy/venv/bin/python
cachedir: .pytest_cache
rootdir: /root/projects/openclaw-proxy
plugins: anyio-4.12.1, asyncio-1.3.0
asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
collecting ... collected 19 items

tests/test_main.py::test_process_stt_calls_whisper PASSED                [  5%]
tests/test_main.py::test_call_openclaw_parses_cli_output PASSED          [ 10%]
tests/test_main.py::test_call_openclaw_raises_on_cli_failure PASSED      [ 15%]
tests/test_main.py::test_call_openclaw_raises_on_empty_payloads PASSED   [ 21%]
tests/test_main.py::test_process_tts_returns_opus_bytes PASSED           [ 26%]
tests/test_main.py::test_process_tts_raises_on_ffmpeg_failure PASSED     [ 31%]
tests/test_main.py::test_session_state_defaults PASSED                   [ 36%]
tests/test_main.py::test_health_endpoint PASSED                          [ 42%]
tests/test_main.py::test_websocket_connect_and_disconnect PASSED         [ 47%]
tests/test_main.py::test_websocket_text_frame_wake_word PASSED           [ 52%]
tests/test_main.py::test_websocket_text_frame_malformed_json PASSED      [ 57%]
tests/test_main.py::test_key_down_starts_listening PASSED                [ 63%]
tests/test_main.py::test_key_up_triggers_pipeline PASSED                 [ 68%]
tests/test_main.py::test_key_up_empty_buffer_skips_pipeline PASSED       [ 73%]
tests/test_main.py::test_binary_frame_while_idle_is_dropped PASSED       [ 78%]
tests/test_main.py::test_binary_frames_are_concatenated PASSED           [ 84%]
tests/test_main.py::test_repeated_key_down_resets_buffer PASSED          [ 89%]
tests/test_main.py::test_key_up_without_key_down_is_ignored PASSED       [ 94%]
tests/test_main.py::test_websocket_binary_frame_full_pipeline PASSED     [100%]

============================== 19 passed in 2.12s ==============================


All 19 tests pass.


## Human-in-the-Loop Protocol

**What changed and why:**
- Added `SessionState` Pydantic model with `idle→listening→processing→idle` state machine
- `handle_text_frame` now routes `key_down`/`wake_word` to start listening and `key_up` to trigger the pipeline
- `handle_binary_frame` no longer triggers the pipeline directly — it only appends to the buffer when in `listening` state
- New `handle_pipeline` drains the buffer and runs STT→LLM→TTS once, with `finally: state.reset()` to guarantee return to idle
- `test_client.py` updated to use the key_down→binary→key_up sequence

**Structural check:**
- No new dependencies. `Literal` added from stdlib `typing`.
- Signature changes: `handle_text_frame(ws, raw)` → `(ws, raw, state, session_id)`, `handle_binary_frame(ws, data, session_id)` → `(ws, data, state)`

**Risk assessment:**
- Most likely breakage: any external code calling the old `handle_binary_frame` or `handle_text_frame` signatures directly. These are only called from `websocket_chat`, which was updated.

**Verification:**
```bash
source venv/bin/activate && python -m pytest tests/ -v
# 19 passed ✓
```


现在可以测试小智机器人了吗，应该怎么操作


The user is asking if they can now test with the XiaoZhi (小智) ESP32 robot and how to operate it. Let me check what's needed to run the server and how the ESP32 connects.

Let me look at the existing setup - the server runs on uvicorn, and the ESP32 would connect via WebSocket to `/chat`. I should check if there are any configuration files or documentation about how to set up the connection.


可以测试了。让我先确认一下运行环境。


No files found


which openclaw 2>&1; which ffmpeg 2>&1; source venv/bin/activate && pip list 2>/dev/null | grep -iE "whisper|edge.tts|fastapi|uvicorn"

Check runtime dependencies


     1→# PLAN.md — openclaw-proxy
     2→
     3→## 项目定位
     4→
     5→轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。
     6→
     7→```
     8→┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
     9→│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
    10→│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
    11→└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
    12→                                  │        ▲                └──────────────┘
    13→                                  ▼        │
    14→                              ┌──────┐ ┌──────┐
    15→                              │ STT  │ │ TTS  │
    16→                              └──────┘ └──────┘
    17→```
    18→
    19→**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
    20→不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。
    21→
    22→## 协议约定
    23→
    24→### 上行（ESP32 → Proxy）
    25→
    26→| 帧类型 | 格式 | 说明 |
    27→|--------|------|------|
    28→| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
    29→| 二进制帧 | Raw Opus bytes | 用户语音音频流 |
    30→
    31→### 下行（Proxy → ESP32）
    32→
    33→| 帧类型 | 格式 | 说明 |
    34→|--------|------|------|
    35→| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
    36→| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
    37→| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |
    38→
    39→## 分阶段实施计划
    40→
    41→---
    42→
    43→### Phase 0 — Mock 骨架搭建 ✅ 已完成
    44→
    45→**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。
    46→
    47→**交付物：**
    48→- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
    49→- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
    50→- [x] `requirements.txt` — 依赖声明
    51→
    52→---
    53→
    54→### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成
    55→
    56→**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。
    57→
    58→**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
    59→Gateway 运行在 `localhost:18789`，CLI 自动连接。
    60→
    61→**交付物：**
    62→- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
    63→- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
    64→- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
    65→- [x] CLI 失败 / 空 payload 的错误处理
    66→- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）
    67→
    68→**管线流程：**
    69→```
    70→Binary frame → process_stt(mock) → send "thinking" state
    71→            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
    72→            → process_tts(mock) → send binary audio
    73→```
    74→
    75→---
    76→
    77→### Phase 1 — 音频缓冲与真实 ESP32 握手
    78→
    79→**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。
    80→
    81→**任务：**
    82→- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
    83→  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
    84→  - 后续二进制帧 → 追加到缓冲区
    85→  - `key_up` → 停止缓冲，将完整音频交给管线
    86→- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
    87→- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
    88→- [ ] 处理边界情况：空音频、超长音频、重复 key_down
    89→- [ ] 补充对应测试用例
    90→
    91→**关键数据结构：**
    92→```python
    93→class SessionState(BaseModel):
    94→    status: Literal["idle", "listening", "processing"] = "idle"
    95→    audio_chunks: list[bytes] = []
    96→```
    97→
    98→---
    99→
   100→### Phase 2 — 真实 STT 接入 ✅ 已完成（2026-03-16）
   101→
   102→**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。
   103→
   104→**决策：** 选用 OpenAI Whisper base 模型，本地 CPU 推理。
   105→
   106→**交付物：**
   107→- [x] Whisper base 模型懒加载（`_get_whisper_model` 单例）
   108→- [x] Opus → tmpfile → Whisper transcribe（`asyncio.to_thread` 非阻塞）
   109→- [x] 固定 `language="zh"` 中文识别
   110→- [x] 测试：mock Whisper 模型验证临时文件创建、语言参数、文本 strip
   111→
   112→---
   113→
   114→### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成
   115→
   116→已通过 `openclaw agent` CLI 完成对接，无需额外工作。
   117→后续可考虑升级为 channel 插件获得更紧密集成。
   118→
   119→---
   120→
   121→### Phase 4 — 真实 TTS 接入 ✅ 已完成（2026-03-16）
   122→
   123→**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。
   124→
   125→**决策：** 选用 edge-tts（微软 Neural TTS），MP3 → ffmpeg → Opus 编码。
   126→
   127→**交付物：**
   128→- [x] edge-tts `Communicate.stream()` 异步收集 MP3 音频
   129→- [x] ffmpeg 管道转码 MP3 → Opus（libopus, 24kbps, 16kHz, mono）
   130→- [x] ffmpeg 失败时抛出 RuntimeError
   131→- [x] 测试：mock edge-tts stream + ffmpeg subprocess，验证 MP3 拼接和错误处理
   132→- [x] 11 个测试全部通过
   133→
   134→> **详细记录见：** `MILESTONE-STT-TTS.md`
   135→
   136→**遗留（留给 Phase 5）：**
   137→- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
   138→- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
   139→
   140→---
   141→
   142→### Phase 5 — 生产加固
   143→
   144→**目标：** 具备长期稳定运行的能力。
   145→
   146→**任务：**
   147→- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
   148→- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
   149→- [ ] 结构化日志：JSON 格式日志，接入日志收集
   150→- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
   151→- [ ] systemd 服务单元文件，支持开机自启和自动重启
   152→- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
   153→- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息
   154→
   155→---
   156→
   157→## 技术栈总览
   158→
   159→| 层 | 当前 | 目标 |
   160→|----|------|------|
   161→| 框架 | FastAPI + uvicorn | 不变 |
   162→| WS 协议 | Starlette WebSocket | 不变 |
   163→| 数据校验 | Pydantic v2 | 不变 |
   164→| STT | Whisper base (本地 CPU) | 不变（可选升级 small/medium） |
   165→| LLM | `openclaw agent` CLI → Gateway :18789 | 同左（可选升级为 channel 插件） |
   166→| TTS | edge-tts → ffmpeg → Opus | 不变（可选 CosyVoice） |
   167→| 音频编解码 | ffmpeg (libopus) | 不变 |
   168→| 部署 | `python main.py` | systemd + nginx |
   169→
   170→## 文件结构规划
   171→
   172→```
   173→openclaw-proxy/
   174→├── main.py              # FastAPI 应用入口、WS 端点、帧路由
   175→├── pipeline/
   176→│   ├── __init__.py
   177→│   ├── stt.py           # process_stt 实现
   178→│   ├── llm.py           # call_openclaw 实现
   179→│   └── tts.py           # process_tts 实现
   180→├── models.py            # Pydantic 模型集中定义
   181→├── session.py           # 连接会话状态管理、音频缓冲
   182→├── config.py            # 配置加载（环境变量 / yaml）
   183→├── tests/
   184→│   ├── test_main.py
   185→│   ├── test_stt.py
   186→│   ├── test_llm.py
   187→│   └── test_tts.py
   188→├── requirements.txt
   189→├── CLAUDE.md
   190→└── PLAN.md
   191→```
   192→
   193→> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。
   194→

# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

---

### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成

**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
Gateway 运行在 `localhost:18789`，CLI 自动连接。

**交付物：**
- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
- [x] CLI 失败 / 空 payload 的错误处理
- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

**管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入 ✅ 已完成（2026-03-16）

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**决策：** 选用 OpenAI Whisper base 模型，本地 CPU 推理。

**交付物：**
- [x] Whisper base 模型懒加载（`_get_whisper_model` 单例）
- [x] Opus → tmpfile → Whisper transcribe（`asyncio.to_thread` 非阻塞）
- [x] 固定 `language="zh"` 中文识别
- [x] 测试：mock Whisper 模型验证临时文件创建、语言参数、文本 strip

---

### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成

已通过 `openclaw agent` CLI 完成对接，无需额外工作。
后续可考虑升级为 channel 插件获得更紧密集成。

---

### Phase 4 — 真实 TTS 接入 ✅ 已完成（2026-03-16）

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**决策：** 选用 edge-tts（微软 Neural TTS），MP3 → ffmpeg → Opus 编码。

**交付物：**
- [x] edge-tts `Communicate.stream()` 异步收集 MP3 音频
- [x] ffmpeg 管道转码 MP3 → Opus（libopus, 24kbps, 16kHz, mono）
- [x] ffmpeg 失败时抛出 RuntimeError
- [x] 测试：mock edge-tts stream + ffmpeg subprocess，验证 MP3 拼接和错误处理
- [x] 11 个测试全部通过

> **详细记录见：** `MILESTONE-STT-TTS.md`

**遗留（留给 Phase 5）：**
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前 | 目标 |
|----|------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | Whisper base (本地 CPU) | 不变（可选升级 small/medium） |
| LLM | `openclaw agent` CLI → Gateway :18789 | 同左（可选升级为 channel 插件） |
| TTS | edge-tts → ffmpeg → Opus | 不变（可选 CosyVoice） |
| 音频编解码 | ffmpeg (libopus) | 不变 |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。


/root/.nvm/versions/node/v22.22.0/bin/openclaw
/usr/bin/ffmpeg
edge-tts                 7.2.7
fastapi                  0.135.1
openai-whisper           20250625
uvicorn                  0.41.0


所有依赖都就绪。以下是测试步骤：

## 启动服务

**终端 1 — 确保 OpenClaw Gateway 运行：**
```bash
# 检查 gateway 是否已启动（默认 :18789）
curl -s http://localhost:18789/health || openclaw gateway start
```

**终端 2 — 启动 proxy：**
```bash
cd /root/projects/openclaw-proxy
source venv/bin/activate
python main.py
# 监听在 ws://0.0.0.0:8000/chat
```

## 测试方式

### 方式 A：用 test_client.py 模拟（不需要 ESP32）

```bash
# 终端 3
source venv/bin/activate
python test_client.py
```

这会模拟 `key_down → 发送假音频 → key_up` 序列。因为是假的 Opus 字节，Whisper 可能识别不出文字或报错，但能验证整条管线是否通畅。

### 方式 B：连接真实小智 ESP32

在小智固件配置中，把 WebSocket 服务器地址改为你这台服务器的地址：

```
ws://<你的服务器IP>:8000/chat
```

操作流程：
1. 小智上电开机，自动连接 WebSocket
2. **按住按键说话** → ESP32 发送 `key_down` + 连续 Opus 音频帧
3. **松开按键** → ESP32 发送 `key_up`
4. Proxy 合并音频 → Whisper 识别 → OpenClaw 回复 → edge-tts 合成 → 语音下发给小智播放

> **注意：** 如果服务器在云上，确保防火墙开放 **8000 端口**（TCP）。如果小智固件要求 `wss://`，需要在前面加一层 nginx 做 TLS 反代。

要现在启动试试吗？


<local-command-caveat>Caveat: The messages below were generated by the user while running local commands. DO NOT respond to these messages or otherwise consider them in your response unless the user explicitly asks you to.</local-command-caveat>


<command-name>/config</command-name>
            <command-message>config</command-message>
            <command-args></command-args>


<local-command-stdout>Set defaultPermissionMode to acceptEdits
Set response language to 中文</local-command-stdout>


<local-command-stdout>Config dialog dismissed</local-command-stdout>


<command-name>/exit</command-name>
            <command-message>exit</command-message>
            <command-args></command-args>


<local-command-stdout>See ya!</local-command-stdout>


<task-notification>
<task-id>b131kkx8g</task-id>
<tool-use-id>toolu_0117GySjms2d6pEAwsBEc6Zp</tool-use-id>
<output-file>/tmp/claude-0/-root-projects-openclaw-proxy/3cc6e826-1199-4350-9754-74df00ea29ab/tasks/b131kkx8g.output</output-file>
<status>killed</status>
<summary>Background command "Start openclaw-proxy server" was stopped</summary>
</task-notification>
Read the output file to retrieve the result: /tmp/claude-0/-root-projects-openclaw-proxy/3cc6e826-1199-4350-9754-74df00ea29ab/tasks/b131kkx8g.output