# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds

# ── Pydantic models ─────────────────────────────────────────────────

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import uuid
from enum import Enum
from shutil import which

from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Mock pipeline ───────────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text (mock).

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.

    TODO (real implementation):
        1. Decode Opus → WAV with ffmpeg-python:
             import ffmpeg
             wav, _ = (ffmpeg
                 .input("pipe:", format="opus")
                 .output("pipe:", format="wav", ar=16000, ac=1)
                 .run(input=audio_bytes, capture_stdout=True, capture_stderr=True))
        2. Or use pydub:
             from pydub import AudioSegment
             seg = AudioSegment.from_file(io.BytesIO(audio_bytes), codec="opus")
             wav = seg.set_frame_rate(16000).set_channels(1).raw_data
        3. Feed WAV bytes to STT (Whisper / FunASR / SenseVoice).
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    return "测试语音输入"

async def call_openclaw(text: str) -> str:
    """Send text to OpenClaw agent and return its reply (mock).

    Args:
        text: User utterance transcribed by STT.

    Returns:
        Agent reply string.

    TODO (real implementation):
        - Communicate with OpenClaw agent in /root/.openclaw
          via its CLI / HTTP / stdin-stdout bridge.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)
    await asyncio.sleep(2)  # simulate thinking latency
    reply = "这是来自 Butler 的测试回复"
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio (mock).

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    TODO (real implementation):
        1. Call TTS engine (edge-tts / CosyVoice / GPT-SoVITS).
        2. Encode output to Opus:
             import ffmpeg
             opus, _ = (ffmpeg
                 .input("pipe:", format="wav")
                 .output("pipe:", format="opus", ar=16000, ac=1)
                 .run(input=wav_bytes, capture_stdout=True))
        3. Stream Opus frames back to ESP32 in ≤960-byte chunks.
    """
    logger.info("TTS: synthesising → %s", text)
    mock_opus = b"\xfc\xff\xfe" * 53  # 159 bytes, resembles Opus silence
    logger.info("TTS: generated %d bytes of audio", len(mock_opus))
    return mock_opus

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM
    reply = await call_openclaw(text)
    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    logger.info("Client connected: %s", peer)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s", peer)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

+# ── Configuration ────────────────────────────────────────────────────

+OPENCLAW_CLI = which("openclaw") or "openclaw"


Now replace the mock `call_openclaw` with the real implementation.


Edit

async def call_openclaw(text: str) -> str:
    """Send text to OpenClaw agent and return its reply (mock).

    Args:
        text: User utterance transcribed by STT.

    Returns:
        Agent reply string.

    TODO (real implementation):
        - Communicate with OpenClaw agent in /root/.openclaw
          via its CLI / HTTP / stdin-stdout bridge.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)
    await asyncio.sleep(2)  # simulate thinking latency
    reply = "这是来自 Butler 的测试回复"
    logger.info("LLM: reply ← %s", reply)
    return reply

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply


The file /root/projects/openclaw-proxy/main.py has been updated successfully.

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import uuid
from enum import Enum
from shutil import which

from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Mock pipeline ───────────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text (mock).

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.

    TODO (real implementation):
        1. Decode Opus → WAV with ffmpeg-python:
             import ffmpeg
             wav, _ = (ffmpeg
                 .input("pipe:", format="opus")
                 .output("pipe:", format="wav", ar=16000, ac=1)
                 .run(input=audio_bytes, capture_stdout=True, capture_stderr=True))
        2. Or use pydub:
             from pydub import AudioSegment
             seg = AudioSegment.from_file(io.BytesIO(audio_bytes), codec="opus")
             wav = seg.set_frame_rate(16000).set_channels(1).raw_data
        3. Feed WAV bytes to STT (Whisper / FunASR / SenseVoice).
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    return "测试语音输入"

async def call_openclaw(text: str) -> str:
    """Send text to OpenClaw agent and return its reply (mock).

    Args:
        text: User utterance transcribed by STT.

    Returns:
        Agent reply string.

    TODO (real implementation):
        - Communicate with OpenClaw agent in /root/.openclaw
          via its CLI / HTTP / stdin-stdout bridge.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)
    await asyncio.sleep(2)  # simulate thinking latency
    reply = "这是来自 Butler 的测试回复"
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio (mock).

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    TODO (real implementation):
        1. Call TTS engine (edge-tts / CosyVoice / GPT-SoVITS).
        2. Encode output to Opus:
             import ffmpeg
             opus, _ = (ffmpeg
                 .input("pipe:", format="wav")
                 .output("pipe:", format="opus", ar=16000, ac=1)
                 .run(input=wav_bytes, capture_stdout=True))
        3. Stream Opus frames back to ESP32 in ≤960-byte chunks.
    """
    logger.info("TTS: synthesising → %s", text)
    mock_opus = b"\xfc\xff\xfe" * 53  # 159 bytes, resembles Opus silence
    logger.info("TTS: generated %d bytes of audio", len(mock_opus))
    return mock_opus

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM
    reply = await call_openclaw(text)
    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    logger.info("Client connected: %s", peer)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s", peer)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-async def call_openclaw(text: str) -> str:

-    """Send text to OpenClaw agent and return its reply (mock).

+async def call_openclaw(text: str, session_id: str) -> str:

+    """Send text to OpenClaw agent via CLI and return its reply.

+        session_id: Per-connection session ID for multi-turn conversation.

-        - Communicate with OpenClaw agent in /root/.openclaw

-          via its CLI / HTTP / stdin-stdout bridge.

+        RuntimeError: If the CLI call fails or returns unexpected output.

-    await asyncio.sleep(2)  # simulate thinking latency

+    proc = await asyncio.create_subprocess_exec(

+        "--timeout", str(OPENCLAW_TIMEOUT),

+        stdout=asyncio.subprocess.PIPE,

+        stderr=asyncio.subprocess.PIPE,

+    stdout, stderr = await proc.communicate()

+        err_msg = stderr.decode(errors="replace").strip()

+        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)

+        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

+    payloads = data.get("result", {}).get("payloads", [])

+        raise RuntimeError("OpenClaw returned empty payloads")

+    reply = payloads[0].get("text", "")


Now update `handle_binary_frame` to pass `session_id` and add thinking heartbeats.


async def handle_binary_frame(ws: WebSocket, data: bytes) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM
    reply = await call_openclaw(text)
    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)


async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import uuid
from enum import Enum
from shutil import which

from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Mock pipeline ───────────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text (mock).

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.

    TODO (real implementation):
        1. Decode Opus → WAV with ffmpeg-python:
             import ffmpeg
             wav, _ = (ffmpeg
                 .input("pipe:", format="opus")
                 .output("pipe:", format="wav", ar=16000, ac=1)
                 .run(input=audio_bytes, capture_stdout=True, capture_stderr=True))
        2. Or use pydub:
             from pydub import AudioSegment
             seg = AudioSegment.from_file(io.BytesIO(audio_bytes), codec="opus")
             wav = seg.set_frame_rate(16000).set_channels(1).raw_data
        3. Feed WAV bytes to STT (Whisper / FunASR / SenseVoice).
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    return "测试语音输入"

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio (mock).

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    TODO (real implementation):
        1. Call TTS engine (edge-tts / CosyVoice / GPT-SoVITS).
        2. Encode output to Opus:
             import ffmpeg
             opus, _ = (ffmpeg
                 .input("pipe:", format="wav")
                 .output("pipe:", format="opus", ar=16000, ac=1)
                 .run(input=wav_bytes, capture_stdout=True))
        3. Stream Opus frames back to ESP32 in ≤960-byte chunks.
    """
    logger.info("TTS: synthesising → %s", text)
    mock_opus = b"\xfc\xff\xfe" * 53  # 159 bytes, resembles Opus silence
    logger.info("TTS: generated %d bytes of audio", len(mock_opus))
    return mock_opus

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM
    reply = await call_openclaw(text)
    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    logger.info("Client connected: %s", peer)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s", peer)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-async def handle_binary_frame(ws: WebSocket, data: bytes) -> None:

+async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:

+        session_id: Per-connection session ID.

+    # Stage 2 — LLM (with periodic thinking heartbeats)

+        """Send thinking heartbeat every 3 seconds while LLM is working."""

+                    StateNotification(status="thinking").model_dump_json()

+    heartbeat_task = asyncio.create_task(heartbeat())

+        reply = await call_openclaw(text, session_id)


Now update the WebSocket handler to create per-connection session IDs and pass them through.


async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    logger.info("Client connected: %s", peer)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s", peer)

async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)


async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"
    logger.info("Client connected: %s (session=%s)", peer, session_id)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data, session_id)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s (session=%s)", peer, session_id)

"""openclaw-proxy — async WebSocket gateway between XiaoZhi ESP32 and OpenClaw agent."""

import asyncio
import json
import logging
import uuid
from enum import Enum
from shutil import which

from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from pydantic import BaseModel, ValidationError

# ── Logging ──────────────────────────────────────────────────────────

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
logger = logging.getLogger("openclaw-proxy")

# ── Configuration ────────────────────────────────────────────────────

OPENCLAW_CLI = which("openclaw") or "openclaw"
OPENCLAW_AGENT_ID = "main"
OPENCLAW_TIMEOUT = 120  # seconds

# ── Pydantic models ─────────────────────────────────────────────────

class HardwareAction(str, Enum):
    """Actions reported by XiaoZhi ESP32 hardware."""

    KEY_DOWN = "key_down"
    KEY_UP = "key_up"
    WAKE_WORD = "wake_word"

class HardwareEvent(BaseModel):
    """Incoming JSON frame from hardware."""

    type: str
    action: HardwareAction

class StateNotification(BaseModel):
    """Outgoing state frame pushed to hardware (e.g. breathing LED trigger)."""

    type: str = "state"
    status: str

class TextReply(BaseModel):
    """Outgoing text reply carrying the LLM response."""

    type: str = "text_reply"
    text: str

# ── Mock pipeline ───────────────────────────────────────────────────

async def process_stt(audio_bytes: bytes) -> str:
    """Convert Opus audio to text (mock).

    Args:
        audio_bytes: Raw Opus-encoded audio from ESP32.

    Returns:
        Transcribed text string.

    TODO (real implementation):
        1. Decode Opus → WAV with ffmpeg-python:
             import ffmpeg
             wav, _ = (ffmpeg
                 .input("pipe:", format="opus")
                 .output("pipe:", format="wav", ar=16000, ac=1)
                 .run(input=audio_bytes, capture_stdout=True, capture_stderr=True))
        2. Or use pydub:
             from pydub import AudioSegment
             seg = AudioSegment.from_file(io.BytesIO(audio_bytes), codec="opus")
             wav = seg.set_frame_rate(16000).set_channels(1).raw_data
        3. Feed WAV bytes to STT (Whisper / FunASR / SenseVoice).
    """
    logger.info("STT: received %d bytes of Opus audio", len(audio_bytes))
    return "测试语音输入"

async def call_openclaw(text: str, session_id: str) -> str:
    """Send text to OpenClaw agent via CLI and return its reply.

    Args:
        text: User utterance transcribed by STT.
        session_id: Per-connection session ID for multi-turn conversation.

    Returns:
        Agent reply string.

    Raises:
        RuntimeError: If the CLI call fails or returns unexpected output.
    """
    logger.info("LLM: sending to OpenClaw → %s", text)

    proc = await asyncio.create_subprocess_exec(
        OPENCLAW_CLI, "agent",
        "--agent", OPENCLAW_AGENT_ID,
        "--session-id", session_id,
        "-m", text,
        "--json",
        "--timeout", str(OPENCLAW_TIMEOUT),
        stdout=asyncio.subprocess.PIPE,
        stderr=asyncio.subprocess.PIPE,
    )
    stdout, stderr = await proc.communicate()

    if proc.returncode != 0:
        err_msg = stderr.decode(errors="replace").strip()
        logger.error("OpenClaw CLI failed (rc=%d): %s", proc.returncode, err_msg)
        raise RuntimeError(f"OpenClaw CLI error: {err_msg}")

    data = json.loads(stdout.decode())
    payloads = data.get("result", {}).get("payloads", [])
    if not payloads:
        raise RuntimeError("OpenClaw returned empty payloads")

    reply = payloads[0].get("text", "")
    logger.info("LLM: reply ← %s", reply)
    return reply

async def process_tts(text: str) -> bytes:
    """Convert text to Opus audio (mock).

    Args:
        text: Text to synthesize.

    Returns:
        Opus-encoded audio bytes.

    TODO (real implementation):
        1. Call TTS engine (edge-tts / CosyVoice / GPT-SoVITS).
        2. Encode output to Opus:
             import ffmpeg
             opus, _ = (ffmpeg
                 .input("pipe:", format="wav")
                 .output("pipe:", format="opus", ar=16000, ac=1)
                 .run(input=wav_bytes, capture_stdout=True))
        3. Stream Opus frames back to ESP32 in ≤960-byte chunks.
    """
    logger.info("TTS: synthesising → %s", text)
    mock_opus = b"\xfc\xff\xfe" * 53  # 159 bytes, resembles Opus silence
    logger.info("TTS: generated %d bytes of audio", len(mock_opus))
    return mock_opus

# ── Frame router ────────────────────────────────────────────────────

async def handle_text_frame(ws: WebSocket, raw: str) -> None:
    """Parse and route an incoming JSON text frame.

    Args:
        ws: Active WebSocket connection.
        raw: Raw text payload received from client.
    """
    try:
        payload = json.loads(raw)
    except json.JSONDecodeError:
        logger.warning("Malformed JSON (ignored): %.120s", raw)
        return

    try:
        event = HardwareEvent(**payload)
    except ValidationError as exc:
        logger.warning("Invalid hardware event: %s", exc.errors())
        return

    logger.info("Hardware event: %s", event.action.value)

    if event.action == HardwareAction.WAKE_WORD:
        ack = StateNotification(status="listening")
        await ws.send_text(ack.model_dump_json())

async def handle_binary_frame(ws: WebSocket, data: bytes, session_id: str) -> None:
    """Run the full STT → LLM → TTS pipeline on an audio binary frame.

    Args:
        ws: Active WebSocket connection.
        data: Raw Opus audio bytes.
        session_id: Per-connection session ID.

    Response sequence pushed to client:
        1. {"type": "state", "status": "thinking"}  — triggers breathing LED
        2. {"type": "text_reply", "text": "..."}     — agent answer
        3. <binary>                                   — Opus audio of the answer
    """
    # Stage 1 — STT
    text = await process_stt(data)

    # Notify hardware: entering thinking phase
    await ws.send_text(StateNotification(status="thinking").model_dump_json())

    # Stage 2 — LLM (with periodic thinking heartbeats)
    async def heartbeat() -> None:
        """Send thinking heartbeat every 3 seconds while LLM is working."""
        while True:
            await asyncio.sleep(3)
            try:
                await ws.send_text(
                    StateNotification(status="thinking").model_dump_json()
                )
            except Exception:
                break

    heartbeat_task = asyncio.create_task(heartbeat())
    try:
        reply = await call_openclaw(text, session_id)
    finally:
        heartbeat_task.cancel()

    await ws.send_text(TextReply(text=reply).model_dump_json())

    # Stage 3 — TTS
    audio = await process_tts(reply)
    logger.info("Pushing %d bytes of audio to client", len(audio))
    await ws.send_bytes(audio)

# ── FastAPI application ─────────────────────────────────────────────

app = FastAPI(title="openclaw-proxy", version="0.1.0")

@app.websocket("/chat")
async def websocket_chat(ws: WebSocket) -> None:
    """Main WebSocket endpoint for XiaoZhi ESP32 hardware."""
    await ws.accept()
    peer = f"{ws.client.host}:{ws.client.port}" if ws.client else "unknown"
    logger.info("Client connected: %s", peer)

    try:
        while True:
            message = await ws.receive()

            if message["type"] == "websocket.disconnect":
                break

            text_data = message.get("text")
            byte_data = message.get("bytes")

            if text_data is not None:
                await handle_text_frame(ws, text_data)
            elif byte_data is not None:
                await handle_binary_frame(ws, byte_data)
    except WebSocketDisconnect:
        pass
    finally:
        logger.info("Client disconnected: %s", peer)

@app.get("/health")
async def health() -> dict[str, str]:
    """Liveness probe."""
    return {"status": "ok"}

# ── Entry point ─────────────────────────────────────────────────────

if __name__ == "__main__":
    import uvicorn

    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

-    logger.info("Client connected: %s", peer)

+    session_id = f"xiaozhi-{uuid.uuid4().hex[:8]}"

+    logger.info("Client connected: %s (session=%s)", peer, session_id)

-                await handle_binary_frame(ws, byte_data)

+                await handle_binary_frame(ws, byte_data, session_id)

-        logger.info("Client disconnected: %s", peer)

+        logger.info("Client disconnected: %s (session=%s)", peer, session_id)


Updated task #4 status


Updated task #5 status


Now update the tests — mock the subprocess so unit tests remain fast and self-contained.


Write

"""Tests for openclaw-proxy WebSocket middleware."""

import json
from unittest.mock import AsyncMock, patch

import pytest
from starlette.testclient import TestClient

from main import app, process_stt, call_openclaw, process_tts

# ── Helpers ──────────────────────────────────────────────────────────

MOCK_OPENCLAW_OUTPUT = json.dumps({
    "status": "ok",
    "result": {
        "payloads": [{"text": "你好！我在这儿。", "mediaUrl": None}],
    },
}).encode()

def _make_mock_process(
    stdout: bytes = MOCK_OPENCLAW_OUTPUT,
    returncode: int = 0,
) -> AsyncMock:
    """Build a mock asyncio.Process with preset stdout/stderr."""
    proc = AsyncMock()
    proc.communicate.return_value = (stdout, b"")
    proc.returncode = returncode
    return proc

# ── Unit tests: pipeline functions ───────────────────────────────────

@pytest.mark.asyncio
async def test_process_stt_returns_mock_text() -> None:
    """STT should return fixed mock string for any input bytes."""
    result = await process_stt(b"\x00" * 80)
    assert result == "测试语音输入"

@pytest.mark.asyncio
async def test_call_openclaw_parses_cli_output() -> None:
    """call_openclaw should parse the CLI JSON and return reply text."""
    mock_proc = _make_mock_process()
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        result = await call_openclaw("你好", "test-session")
    assert result == "你好！我在这儿。"

@pytest.mark.asyncio
async def test_call_openclaw_raises_on_cli_failure() -> None:
    """call_openclaw should raise RuntimeError when CLI exits non-zero."""
    mock_proc = _make_mock_process(stdout=b"", returncode=1)
    mock_proc.communicate.return_value = (b"", b"some error")
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with pytest.raises(RuntimeError, match="OpenClaw CLI error"):
            await call_openclaw("你好", "test-session")

@pytest.mark.asyncio
async def test_call_openclaw_raises_on_empty_payloads() -> None:
    """call_openclaw should raise RuntimeError on empty payloads."""
    empty = json.dumps({"status": "ok", "result": {"payloads": []}}).encode()
    mock_proc = _make_mock_process(stdout=empty)
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with pytest.raises(RuntimeError, match="empty payloads"):
            await call_openclaw("你好", "test-session")

@pytest.mark.asyncio
async def test_process_tts_returns_bytes() -> None:
    """TTS mock should return non-empty bytes."""
    result = await process_tts("测试")
    assert isinstance(result, bytes)
    assert len(result) > 0

# ── Integration tests: HTTP & WebSocket ──────────────────────────────

def test_health_endpoint() -> None:
    """GET /health should return 200 with ok status."""
    client = TestClient(app)
    resp = client.get("/health")
    assert resp.status_code == 200
    assert resp.json() == {"status": "ok"}

def test_websocket_connect_and_disconnect() -> None:
    """Client should be able to connect to /chat and cleanly disconnect."""
    client = TestClient(app)
    with client.websocket_connect("/chat"):
        pass

def test_websocket_text_frame_wake_word() -> None:
    """Sending a wake_word JSON should get back a 'listening' state."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["type"] == "state"
        assert resp["status"] == "listening"

def test_websocket_text_frame_malformed_json() -> None:
    """Malformed JSON should not crash the connection."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text("not-valid-json{{{")
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["status"] == "listening"

def test_websocket_binary_frame_full_pipeline() -> None:
    """Binary audio should trigger STT → LLM → TTS pipeline.

    Expected response sequence:
      1. {"type": "state", "status": "thinking"}
      2. {"type": "text_reply", "text": "..."}
      3. binary bytes (TTS audio)
    """
    mock_proc = _make_mock_process()
    client = TestClient(app)
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with client.websocket_connect("/chat") as ws:
            ws.send_bytes(b"\x00" * 80)

            # 1) thinking state
            msg1 = json.loads(ws.receive_text())
            assert msg1 == {"type": "state", "status": "thinking"}

            # 2) text reply
            msg2 = json.loads(ws.receive_text())
            assert msg2["type"] == "text_reply"
            assert msg2["text"] == "你好！我在这儿。"

            # 3) TTS audio bytes
            audio = ws.receive_bytes()
            assert isinstance(audio, bytes)
            assert len(audio) > 0


The file /root/projects/openclaw-proxy/tests/test_main.py has been updated successfully.

"""Tests for openclaw-proxy WebSocket middleware."""

import json
from unittest.mock import AsyncMock, patch

import pytest
from starlette.testclient import TestClient

from main import app, process_stt, call_openclaw, process_tts

# ── Helpers ──────────────────────────────────────────────────────────

MOCK_OPENCLAW_OUTPUT = json.dumps({
    "status": "ok",
    "result": {
        "payloads": [{"text": "你好！我在这儿。", "mediaUrl": None}],
    },
}).encode()

def _make_mock_process(
    stdout: bytes = MOCK_OPENCLAW_OUTPUT,
    returncode: int = 0,
) -> AsyncMock:
    """Build a mock asyncio.Process with preset stdout/stderr."""
    proc = AsyncMock()
    proc.communicate.return_value = (stdout, b"")
    proc.returncode = returncode
    return proc

# ── Unit tests: pipeline functions ───────────────────────────────────

@pytest.mark.asyncio
async def test_process_stt_returns_mock_text() -> None:
    """STT should return fixed mock string for any input bytes."""
    result = await process_stt(b"\x00" * 80)
    assert result == "测试语音输入"

@pytest.mark.asyncio
async def test_call_openclaw_parses_cli_output() -> None:
    """call_openclaw should parse the CLI JSON and return reply text."""
    mock_proc = _make_mock_process()
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        result = await call_openclaw("你好", "test-session")
    assert result == "你好！我在这儿。"

@pytest.mark.asyncio
async def test_call_openclaw_raises_on_cli_failure() -> None:
    """call_openclaw should raise RuntimeError when CLI exits non-zero."""
    mock_proc = _make_mock_process(stdout=b"", returncode=1)
    mock_proc.communicate.return_value = (b"", b"some error")
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with pytest.raises(RuntimeError, match="OpenClaw CLI error"):
            await call_openclaw("你好", "test-session")

@pytest.mark.asyncio
async def test_call_openclaw_raises_on_empty_payloads() -> None:
    """call_openclaw should raise RuntimeError on empty payloads."""
    empty = json.dumps({"status": "ok", "result": {"payloads": []}}).encode()
    mock_proc = _make_mock_process(stdout=empty)
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with pytest.raises(RuntimeError, match="empty payloads"):
            await call_openclaw("你好", "test-session")

@pytest.mark.asyncio
async def test_process_tts_returns_bytes() -> None:
    """TTS mock should return non-empty bytes."""
    result = await process_tts("测试")
    assert isinstance(result, bytes)
    assert len(result) > 0

# ── Integration tests: HTTP & WebSocket ──────────────────────────────

def test_health_endpoint() -> None:
    """GET /health should return 200 with ok status."""
    client = TestClient(app)
    resp = client.get("/health")
    assert resp.status_code == 200
    assert resp.json() == {"status": "ok"}

def test_websocket_connect_and_disconnect() -> None:
    """Client should be able to connect to /chat and cleanly disconnect."""
    client = TestClient(app)
    with client.websocket_connect("/chat"):
        pass

def test_websocket_text_frame_wake_word() -> None:
    """Sending a wake_word JSON should get back a 'listening' state."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["type"] == "state"
        assert resp["status"] == "listening"

def test_websocket_text_frame_malformed_json() -> None:
    """Malformed JSON should not crash the connection."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text("not-valid-json{{{")
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["status"] == "listening"

def test_websocket_binary_frame_full_pipeline() -> None:
    """Binary audio should trigger STT → LLM → TTS pipeline.

    Expected response sequence:
      1. {"type": "state", "status": "thinking"}
      2. {"type": "text_reply", "text": "..."}
      3. binary bytes (TTS audio)
    """
    mock_proc = _make_mock_process()
    client = TestClient(app)
    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):
        with client.websocket_connect("/chat") as ws:
            ws.send_bytes(b"\x00" * 80)

            # 1) thinking state
            msg1 = json.loads(ws.receive_text())
            assert msg1 == {"type": "state", "status": "thinking"}

            # 2) text reply
            msg2 = json.loads(ws.receive_text())
            assert msg2["type"] == "text_reply"
            assert msg2["text"] == "你好！我在这儿。"

            # 3) TTS audio bytes
            audio = ws.receive_bytes()
            assert isinstance(audio, bytes)
            assert len(audio) > 0

+from unittest.mock import AsyncMock, patch

-# ── Unit tests: mock pipeline functions ──────────────────────────────

+# ── Helpers ──────────────────────────────────────────────────────────

+        "payloads": [{"text": "你好！我在这儿。", "mediaUrl": None}],

+    stdout: bytes = MOCK_OPENCLAW_OUTPUT,

+    """Build a mock asyncio.Process with preset stdout/stderr."""

+    proc.communicate.return_value = (stdout, b"")

+# ── Unit tests: pipeline functions ───────────────────────────────────

-async def test_call_openclaw_returns_mock_reply() -> None:

-    """LLM mock should return the fixed Butler reply."""

-    assert result == "这是来自 Butler 的测试回复"

+async def test_call_openclaw_parses_cli_output() -> None:

+    """call_openclaw should parse the CLI JSON and return reply text."""

+    with patch("main.asyncio.create_subprocess_exec", return_value=mock_proc):

+        result = await call_openclaw("你好", "test-session")

+async def test_call_openclaw_raises_on_cli_failure() -> None:

+    """call_openclaw should raise RuntimeError when CLI exits non-zero."""

+    mock_proc = _make_mock_process(stdout=b"", returncode=1)

+    mock_proc.communicate.return_value = (b"", b"some error")

+        with pytest.raises(RuntimeError, match="OpenClaw CLI error"):

+            await call_openclaw("你好", "test-session")

+async def test_call_openclaw_raises_on_empty_payloads() -> None:

+    """call_openclaw should raise RuntimeError on empty payloads."""

+    empty = json.dumps({"status": "ok", "result": {"payloads": []}}).encode()

+    mock_proc = _make_mock_process(stdout=empty)

+        with pytest.raises(RuntimeError, match="empty payloads"):

-# ── Integration tests: WebSocket endpoint ────────────────────────────

+# ── Integration tests: HTTP & WebSocket ──────────────────────────────

-    with client.websocket_connect("/chat") as ws:

-        # Just connecting and disconnecting should not raise

+    with client.websocket_connect("/chat"):

-        # Connection should stay alive — send a valid frame after

-    """Sending binary audio should trigger STT → LLM → TTS pipeline.

+    """Binary audio should trigger STT → LLM → TTS pipeline.

-      1. {"type": "state", "status": "thinking"}   — UX heartbeat

-      2. {"type": "text_reply", "text": "..."}      — LLM reply text

-      3. binary bytes                                — TTS audio

+      1. {"type": "state", "status": "thinking"}

+        with client.websocket_connect("/chat") as ws:

+            ws.send_bytes(b"\x00" * 80)

-        msg1 = json.loads(ws.receive_text())

-        assert msg1 == {"type": "state", "status": "thinking"}

+            msg1 = json.loads(ws.receive_text())

+            assert msg1 == {"type": "state", "status": "thinking"}

-        msg2 = json.loads(ws.receive_text())

-        assert msg2["type"] == "text_reply"

-        assert msg2["text"] == "这是来自 Butler 的测试回复"

+            msg2 = json.loads(ws.receive_text())

+            assert msg2["type"] == "text_reply"

+            assert msg2["text"] == "你好！我在这儿。"

-        assert isinstance(audio, bytes)

+            assert isinstance(audio, bytes)

"""Tests for openclaw-proxy WebSocket middleware."""

import json

import pytest
from starlette.testclient import TestClient

from main import app, process_stt, call_openclaw, process_tts

# ── Unit tests: mock pipeline functions ──────────────────────────────

@pytest.mark.asyncio
async def test_process_stt_returns_mock_text() -> None:
    """STT should return fixed mock string for any input bytes."""
    result = await process_stt(b"\x00" * 80)
    assert result == "测试语音输入"

@pytest.mark.asyncio
async def test_call_openclaw_returns_mock_reply() -> None:
    """LLM mock should return the fixed Butler reply."""
    result = await call_openclaw("你好")
    assert result == "这是来自 Butler 的测试回复"

@pytest.mark.asyncio
async def test_process_tts_returns_bytes() -> None:
    """TTS mock should return non-empty bytes."""
    result = await process_tts("测试")
    assert isinstance(result, bytes)
    assert len(result) > 0

# ── Integration tests: WebSocket endpoint ────────────────────────────

def test_health_endpoint() -> None:
    """GET /health should return 200 with ok status."""
    client = TestClient(app)
    resp = client.get("/health")
    assert resp.status_code == 200
    assert resp.json() == {"status": "ok"}

def test_websocket_connect_and_disconnect() -> None:
    """Client should be able to connect to /chat and cleanly disconnect."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        # Just connecting and disconnecting should not raise
        pass

def test_websocket_text_frame_wake_word() -> None:
    """Sending a wake_word JSON should get back a 'listening' state."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["type"] == "state"
        assert resp["status"] == "listening"

def test_websocket_text_frame_malformed_json() -> None:
    """Malformed JSON should not crash the connection."""
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_text("not-valid-json{{{")
        # Connection should stay alive — send a valid frame after
        ws.send_text(json.dumps({
            "type": "hardware_event",
            "action": "wake_word",
        }))
        resp = json.loads(ws.receive_text())
        assert resp["status"] == "listening"

def test_websocket_binary_frame_full_pipeline() -> None:
    """Sending binary audio should trigger STT → LLM → TTS pipeline.

    Expected sequence of responses:
      1. {"type": "state", "status": "thinking"}   — UX heartbeat
      2. {"type": "text_reply", "text": "..."}      — LLM reply text
      3. binary bytes                                — TTS audio
    """
    client = TestClient(app)
    with client.websocket_connect("/chat") as ws:
        ws.send_bytes(b"\x00" * 80)

        # 1) thinking state
        msg1 = json.loads(ws.receive_text())
        assert msg1 == {"type": "state", "status": "thinking"}

        # 2) text reply
        msg2 = json.loads(ws.receive_text())
        assert msg2["type"] == "text_reply"
        assert msg2["text"] == "这是来自 Butler 的测试回复"

        # 3) TTS audio bytes
        audio = ws.receive_bytes()
        assert isinstance(audio, bytes)
        assert len(audio) > 0


Bash

source venv/bin/activate && python -m pytest tests/ -v 2>&1

Run all tests


============================= test session starts ==============================
platform linux -- Python 3.11.6, pytest-9.0.2, pluggy-1.6.0 -- /root/projects/openclaw-proxy/venv/bin/python
cachedir: .pytest_cache
rootdir: /root/projects/openclaw-proxy
plugins: anyio-4.12.1, asyncio-1.3.0
asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
collecting ... collected 10 items

tests/test_main.py::test_process_stt_returns_mock_text PASSED            [ 10%]
tests/test_main.py::test_call_openclaw_parses_cli_output PASSED          [ 20%]
tests/test_main.py::test_call_openclaw_raises_on_cli_failure PASSED      [ 30%]
tests/test_main.py::test_call_openclaw_raises_on_empty_payloads PASSED   [ 40%]
tests/test_main.py::test_process_tts_returns_bytes PASSED                [ 50%]
tests/test_main.py::test_health_endpoint PASSED                          [ 60%]
tests/test_main.py::test_websocket_connect_and_disconnect PASSED         [ 70%]
tests/test_main.py::test_websocket_text_frame_wake_word PASSED           [ 80%]
tests/test_main.py::test_websocket_text_frame_malformed_json PASSED      [ 90%]
tests/test_main.py::test_websocket_binary_frame_full_pipeline PASSED     [100%]

============================== 10 passed in 0.34s ==============================


10/10 全通过。现在更新 PLAN.md。


Updated task #6 status


     1→# PLAN.md — openclaw-proxy
     2→
     3→## 项目定位
     4→
     5→轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。
     6→
     7→```
     8→┌──────────────┐  Opus/JSON   ┌──────────────────┐  text   ┌──────────────┐
     9→│  XiaoZhi     │◄────────────►│  openclaw-proxy  │◄───────►│  OpenClaw    │
    10→│  ESP32       │  WebSocket   │  (FastAPI:8000)   │         │  Agent       │
    11→└──────────────┘              └──────────────────┘         └──────────────┘
    12→                                  │        ▲
    13→                                  ▼        │
    14→                              ┌──────┐ ┌──────┐
    15→                              │ STT  │ │ TTS  │
    16→                              └──────┘ └──────┘
    17→```
    18→
    19→## 协议约定
    20→
    21→### 上行（ESP32 → Proxy）
    22→
    23→| 帧类型 | 格式 | 说明 |
    24→|--------|------|------|
    25→| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
    26→| 二进制帧 | Raw Opus bytes | 用户语音音频流 |
    27→
    28→### 下行（Proxy → ESP32）
    29→
    30→| 帧类型 | 格式 | 说明 |
    31→|--------|------|------|
    32→| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
    33→| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
    34→| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |
    35→
    36→## 分阶段实施计划
    37→
    38→---
    39→
    40→### Phase 0 — Mock 骨架搭建 ✅ 已完成
    41→
    42→**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。
    43→
    44→**交付物：**
    45→- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
    46→- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
    47→- [x] `requirements.txt` — 依赖声明
    48→
    49→**当前管线流程：**
    50→```
    51→Binary frame → process_stt(mock) → send "thinking" state
    52→            → call_openclaw(mock, 2s delay) → send text_reply
    53→            → process_tts(mock) → send binary audio
    54→```
    55→
    56→---
    57→
    58→### Phase 1 — 音频缓冲与真实 ESP32 握手
    59→
    60→**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。
    61→
    62→**任务：**
    63→- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
    64→  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
    65→  - 后续二进制帧 → 追加到缓冲区
    66→  - `key_up` → 停止缓冲，将完整音频交给管线
    67→- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
    68→- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
    69→- [ ] 处理边界情况：空音频、超长音频、重复 key_down
    70→- [ ] 补充对应测试用例
    71→
    72→**关键数据结构：**
    73→```python
    74→class SessionState(BaseModel):
    75→    status: Literal["idle", "listening", "processing"] = "idle"
    76→    audio_chunks: list[bytes] = []
    77→```
    78→
    79→---
    80→
    81→### Phase 2 — 真实 STT 接入
    82→
    83→**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。
    84→
    85→**任务：**
    86→- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
    87→- [ ] 实现 Opus → WAV 转码
    88→  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
    89→  - 方案 B：`pydub` + `ffmpeg` 系统命令
    90→- [ ] 实现真实 STT 调用，返回识别文本
    91→- [ ] 处理空音频 / 识别失败的降级逻辑
    92→- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）
    93→
    94→**依赖：** 服务器需安装 `ffmpeg` 系统包。
    95→
    96→---
    97→
    98→### Phase 3 — OpenClaw 智能体对接
    99→
   100→**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。
   101→
   102→**任务：**
   103→- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
   104→  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
   105→  - 方式 B：CLI 子进程调用
   106→  - 方式 C：直接导入 Python 模块
   107→- [ ] 实现真实调用，支持超时和错误处理
   108→- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
   109→- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
   110→- [ ] 补充测试
   111→
   112→---
   113→
   114→### Phase 4 — 真实 TTS 接入
   115→
   116→**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。
   117→
   118→**任务：**
   119→- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
   120→- [ ] 实现文本 → WAV → Opus 合成管线
   121→- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
   122→- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
   123→- [ ] 补充测试
   124→
   125→**分块推流时序：**
   126→```
   127→send state("speaking")
   128→for chunk in opus_chunks:
   129→    send binary(chunk)
   130→    await asyncio.sleep(frame_duration)
   131→send state("idle")
   132→```
   133→
   134→---
   135→
   136→### Phase 5 — 生产加固
   137→
   138→**目标：** 具备长期稳定运行的能力。
   139→
   140→**任务：**
   141→- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
   142→- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
   143→- [ ] 结构化日志：JSON 格式日志，接入日志收集
   144→- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
   145→- [ ] systemd 服务单元文件，支持开机自启和自动重启
   146→- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
   147→- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息
   148→
   149→---
   150→
   151→## 技术栈总览
   152→
   153→| 层 | 当前（Phase 0） | 目标 |
   154→|----|-----------------|------|
   155→| 框架 | FastAPI + uvicorn | 不变 |
   156→| WS 协议 | Starlette WebSocket | 不变 |
   157→| 数据校验 | Pydantic v2 | 不变 |
   158→| STT | mock | FunASR / SenseVoice / Whisper |
   159→| LLM | mock (2s sleep) | OpenClaw agent |
   160→| TTS | mock (静音帧) | edge-tts / CosyVoice |
   161→| 音频编解码 | 无 | ffmpeg-python |
   162→| 部署 | `python main.py` | systemd + nginx |
   163→
   164→## 文件结构规划
   165→
   166→```
   167→openclaw-proxy/
   168→├── main.py              # FastAPI 应用入口、WS 端点、帧路由
   169→├── pipeline/
   170→│   ├── __init__.py
   171→│   ├── stt.py           # process_stt 实现
   172→│   ├── llm.py           # call_openclaw 实现
   173→│   └── tts.py           # process_tts 实现
   174→├── models.py            # Pydantic 模型集中定义
   175→├── session.py           # 连接会话状态管理、音频缓冲
   176→├── config.py            # 配置加载（环境变量 / yaml）
   177→├── tests/
   178→│   ├── test_main.py
   179→│   ├── test_stt.py
   180→│   ├── test_llm.py
   181→│   └── test_tts.py
   182→├── requirements.txt
   183→├── CLAUDE.md
   184→└── PLAN.md
   185→```
   186→
   187→> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。
   188→

# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐  text   ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │◄───────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │         │  Agent       │
└──────────────┘              └──────────────────┘         └──────────────┘
                                  │        ▲
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

**当前管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(mock, 2s delay) → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**任务：**
- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
- [ ] 实现 Opus → WAV 转码
  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
  - 方案 B：`pydub` + `ffmpeg` 系统命令
- [ ] 实现真实 STT 调用，返回识别文本
- [ ] 处理空音频 / 识别失败的降级逻辑
- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）

**依赖：** 服务器需安装 `ffmpeg` 系统包。

---

### Phase 3 — OpenClaw 智能体对接

**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

**任务：**
- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
  - 方式 B：CLI 子进程调用
  - 方式 C：直接导入 Python 模块
- [ ] 实现真实调用，支持超时和错误处理
- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
- [ ] 补充测试

---

### Phase 4 — 真实 TTS 接入

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**任务：**
- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
- [ ] 实现文本 → WAV → Opus 合成管线
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
- [ ] 补充测试

**分块推流时序：**
```
send state("speaking")
for chunk in opus_chunks:
    send binary(chunk)
    await asyncio.sleep(frame_duration)
send state("idle")
```

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前（Phase 0） | 目标 |
|----|-----------------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | mock | FunASR / SenseVoice / Whisper |
| LLM | mock (2s sleep) | OpenClaw agent |
| TTS | mock (静音帧) | edge-tts / CosyVoice |
| 音频编解码 | 无 | ffmpeg-python |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。


Edit

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐  text   ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │◄───────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │         │  Agent       │
└──────────────┘              └──────────────────┘         └──────────────┘
                                  │        ▲
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。


The file /root/projects/openclaw-proxy/PLAN.md has been updated successfully.

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。

# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐  text   ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │◄───────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │         │  Agent       │
└──────────────┘              └──────────────────┘         └──────────────┘
                                  │        ▲
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

**当前管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(mock, 2s delay) → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**任务：**
- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
- [ ] 实现 Opus → WAV 转码
  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
  - 方案 B：`pydub` + `ffmpeg` 系统命令
- [ ] 实现真实 STT 调用，返回识别文本
- [ ] 处理空音频 / 识别失败的降级逻辑
- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）

**依赖：** 服务器需安装 `ffmpeg` 系统包。

---

### Phase 3 — OpenClaw 智能体对接

**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

**任务：**
- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
  - 方式 B：CLI 子进程调用
  - 方式 C：直接导入 Python 模块
- [ ] 实现真实调用，支持超时和错误处理
- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
- [ ] 补充测试

---

### Phase 4 — 真实 TTS 接入

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**任务：**
- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
- [ ] 实现文本 → WAV → Opus 合成管线
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
- [ ] 补充测试

**分块推流时序：**
```
send state("speaking")
for chunk in opus_chunks:
    send binary(chunk)
    await asyncio.sleep(frame_duration)
send state("idle")
```

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前（Phase 0） | 目标 |
|----|-----------------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | mock | FunASR / SenseVoice / Whisper |
| LLM | mock (2s sleep) | OpenClaw agent |
| TTS | mock (静音帧) | edge-tts / CosyVoice |
| 音频编解码 | 无 | ffmpeg-python |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。

-┌──────────────┐  Opus/JSON   ┌──────────────────┐  text   ┌──────────────┐

-│  XiaoZhi     │◄────────────►│  openclaw-proxy  │◄───────►│  OpenClaw    │

-│  ESP32       │  WebSocket   │  (FastAPI:8000)   │         │  Agent       │

+┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐

+│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │

+│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │

+└──────────────┘              └──────────────────┘◄────────│  (:18789)    │

+**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，

+不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。


### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

**当前管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(mock, 2s delay) → send text_reply
            → process_tts(mock) → send binary audio
```

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

---

### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成

**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
Gateway 运行在 `localhost:18789`，CLI 自动连接。

**交付物：**
- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
- [x] CLI 失败 / 空 payload 的错误处理
- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

**管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
            → process_tts(mock) → send binary audio
```


### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

---

### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成

**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
Gateway 运行在 `localhost:18789`，CLI 自动连接。

**交付物：**
- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
- [x] CLI 失败 / 空 payload 的错误处理
- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

**管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
            → process_tts(mock) → send binary audio
```

# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

**当前管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(mock, 2s delay) → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**任务：**
- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
- [ ] 实现 Opus → WAV 转码
  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
  - 方案 B：`pydub` + `ffmpeg` 系统命令
- [ ] 实现真实 STT 调用，返回识别文本
- [ ] 处理空音频 / 识别失败的降级逻辑
- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）

**依赖：** 服务器需安装 `ffmpeg` 系统包。

---

### Phase 3 — OpenClaw 智能体对接

**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

**任务：**
- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
  - 方式 B：CLI 子进程调用
  - 方式 C：直接导入 Python 模块
- [ ] 实现真实调用，支持超时和错误处理
- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
- [ ] 补充测试

---

### Phase 4 — 真实 TTS 接入

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**任务：**
- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
- [ ] 实现文本 → WAV → Opus 合成管线
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
- [ ] 补充测试

**分块推流时序：**
```
send state("speaking")
for chunk in opus_chunks:
    send binary(chunk)
    await asyncio.sleep(frame_duration)
send state("idle")
```

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前（Phase 0） | 目标 |
|----|-----------------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | mock | FunASR / SenseVoice / Whisper |
| LLM | mock (2s sleep) | OpenClaw agent |
| TTS | mock (静音帧) | edge-tts / CosyVoice |
| 音频编解码 | 无 | ffmpeg-python |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。

-**当前管线流程：**

+**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

+**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。

+Gateway 运行在 `localhost:18789`，CLI 自动连接。

+- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI

+- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话

+- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃

+- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

-            → call_openclaw(mock, 2s delay) → send text_reply

+            → call_openclaw(CLI) → heartbeat every 3s → send text_reply


### Phase 3 — OpenClaw 智能体对接

**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

**任务：**
- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
  - 方式 B：CLI 子进程调用
  - 方式 C：直接导入 Python 模块
- [ ] 实现真实调用，支持超时和错误处理
- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
- [ ] 补充测试

### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成

已通过 `openclaw agent` CLI 完成对接，无需额外工作。
后续可考虑升级为 channel 插件获得更紧密集成。


### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成

已通过 `openclaw agent` CLI 完成对接，无需额外工作。
后续可考虑升级为 channel 插件获得更紧密集成。

# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

---

### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成

**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
Gateway 运行在 `localhost:18789`，CLI 自动连接。

**交付物：**
- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
- [x] CLI 失败 / 空 payload 的错误处理
- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

**管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**任务：**
- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
- [ ] 实现 Opus → WAV 转码
  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
  - 方案 B：`pydub` + `ffmpeg` 系统命令
- [ ] 实现真实 STT 调用，返回识别文本
- [ ] 处理空音频 / 识别失败的降级逻辑
- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）

**依赖：** 服务器需安装 `ffmpeg` 系统包。

---

### Phase 3 — OpenClaw 智能体对接

**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

**任务：**
- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）
  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）
  - 方式 B：CLI 子进程调用
  - 方式 C：直接导入 Python 模块
- [ ] 实现真实调用，支持超时和错误处理
- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）
- [ ] 支持流式响应（如果 OpenClaw 支持 streaming）
- [ ] 补充测试

---

### Phase 4 — 真实 TTS 接入

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**任务：**
- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
- [ ] 实现文本 → WAV → Opus 合成管线
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
- [ ] 补充测试

**分块推流时序：**
```
send state("speaking")
for chunk in opus_chunks:
    send binary(chunk)
    await asyncio.sleep(frame_duration)
send state("idle")
```

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前（Phase 0） | 目标 |
|----|-----------------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | mock | FunASR / SenseVoice / Whisper |
| LLM | mock (2s sleep) | OpenClaw agent |
| TTS | mock (静音帧) | edge-tts / CosyVoice |
| 音频编解码 | 无 | ffmpeg-python |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。

+### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成

-**目标：** 将 `call_openclaw` 从 mock 替换为真实 OpenClaw 调用。

+后续可考虑升级为 channel 插件获得更紧密集成。

-- [ ] 调研 OpenClaw 交互方式（查看 `/root/.openclaw/openclaw.json` 配置）

-  - 方式 A：HTTP API（如果 OpenClaw 暴露了本地 HTTP 端口）

-  - 方式 B：CLI 子进程调用

-  - 方式 C：直接导入 Python 模块

-- [ ] 实现真实调用，支持超时和错误处理

-- [ ] 思考期间每 500ms 发送一次 `thinking` 心跳（长时间推理时保持硬件活跃）


| LLM | mock (2s sleep) | OpenClaw agent |

| LLM | `openclaw agent` CLI → Gateway :18789 | 同左（可选升级为 channel 插件） |


# PLAN.md — openclaw-proxy

## 项目定位

轻量级异步 WebSocket 网关，部署在 Linux 云服务器上，负责在 **小智 ESP32 语音硬件** 与 **OpenClaw 智能体** (`/root/.openclaw/`) 之间做协议翻译和管线编排。

```
┌──────────────┐  Opus/JSON   ┌──────────────────┐   CLI    ┌──────────────┐
│  XiaoZhi     │◄────────────►│  openclaw-proxy  │────────►│  OpenClaw    │
│  ESP32       │  WebSocket   │  (FastAPI:8000)   │  stdout  │  Gateway     │
└──────────────┘              └──────────────────┘◄────────│  (:18789)    │
                                  │        ▲                └──────────────┘
                                  ▼        │
                              ┌──────┐ ┌──────┐
                              │ STT  │ │ TTS  │
                              └──────┘ └──────┘
```

**对接方式决策（2026-03-15）：** 采用方案 B — 通过 `openclaw agent` CLI 调用 gateway，
不修改 `.openclaw/` 目录，不注册 channel 插件。优先跑通演示。

## 协议约定

### 上行（ESP32 → Proxy）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "hardware_event", "action": "wake_word\|key_down\|key_up"}` | 硬件状态事件 |
| 二进制帧 | Raw Opus bytes | 用户语音音频流 |

### 下行（Proxy → ESP32）

| 帧类型 | 格式 | 说明 |
|--------|------|------|
| 文本帧 | `{"type": "state", "status": "listening\|thinking\|speaking"}` | 控制硬件 LED 状态 |
| 文本帧 | `{"type": "text_reply", "text": "..."}` | LLM 文本回复 |
| 二进制帧 | Raw Opus bytes | TTS 合成音频流 |

## 分阶段实施计划

---

### Phase 0 — Mock 骨架搭建 ✅ 已完成

**目标：** 跑通 WebSocket 握手和完整数据通路，用假数据验证时序正确。

**交付物：**
- [x] `main.py` — FastAPI + WS `/chat` 端点，帧路由，三段 mock 管线
- [x] `tests/test_main.py` — 8 个测试全部通过（3 单元 + 5 集成）
- [x] `requirements.txt` — 依赖声明

---

### Phase 0.5 — OpenClaw 真实对接 ✅ 已完成

**目标：** 将 `call_openclaw` 从 mock 替换为真实 CLI 调用，跑通文本对话。

**决策：** 采用 `openclaw agent --agent main --session-id {id} -m {text} --json` 命令。
Gateway 运行在 `localhost:18789`，CLI 自动连接。

**交付物：**
- [x] `call_openclaw` 通过 `asyncio.create_subprocess_exec` 调用 CLI
- [x] 每个 WebSocket 连接分配独立 `session_id`，支持多轮对话
- [x] LLM 推理期间每 3 秒发送 thinking 心跳，保持硬件活跃
- [x] CLI 失败 / 空 payload 的错误处理
- [x] `tests/test_main.py` — 10 个测试全部通过（mock subprocess）

**管线流程：**
```
Binary frame → process_stt(mock) → send "thinking" state
            → call_openclaw(CLI) → heartbeat every 3s → send text_reply
            → process_tts(mock) → send binary audio
```

---

### Phase 1 — 音频缓冲与真实 ESP32 握手

**目标：** 支持 ESP32 的真实交互模式——用户按住按键持续说话，松开后触发管线。

**任务：**
- [ ] 实现 `AudioBuffer` 类，按连接维护音频缓冲区
  - `key_down` / `wake_word` → 开始缓冲，进入 "listening" 状态
  - 后续二进制帧 → 追加到缓冲区
  - `key_up` → 停止缓冲，将完整音频交给管线
- [ ] 重构 `handle_binary_frame` 为缓冲追加逻辑
- [ ] 新增 `handle_pipeline(ws, audio_buffer)` 统一编排三段管线
- [ ] 处理边界情况：空音频、超长音频、重复 key_down
- [ ] 补充对应测试用例

**关键数据结构：**
```python
class SessionState(BaseModel):
    status: Literal["idle", "listening", "processing"] = "idle"
    audio_chunks: list[bytes] = []
```

---

### Phase 2 — 真实 STT 接入

**目标：** 将 `process_stt` 从 mock 替换为真实语音识别。

**任务：**
- [ ] 选型确认（FunASR / SenseVoice / Whisper API 三选一）
- [ ] 实现 Opus → WAV 转码
  - 方案 A：`ffmpeg-python` 子进程管道（推荐，无需编译 C 扩展）
  - 方案 B：`pydub` + `ffmpeg` 系统命令
- [ ] 实现真实 STT 调用，返回识别文本
- [ ] 处理空音频 / 识别失败的降级逻辑
- [ ] 补充测试（mock 外部服务，验证转码和调用逻辑）

**依赖：** 服务器需安装 `ffmpeg` 系统包。

---

### Phase 3 — ~~OpenClaw 智能体对接~~ ✅ 已在 Phase 0.5 完成

已通过 `openclaw agent` CLI 完成对接，无需额外工作。
后续可考虑升级为 channel 插件获得更紧密集成。

---

### Phase 4 — 真实 TTS 接入

**目标：** 将 `process_tts` 从 mock 替换为真实语音合成。

**任务：**
- [ ] 选型确认（edge-tts / CosyVoice / GPT-SoVITS 三选一）
- [ ] 实现文本 → WAV → Opus 合成管线
- [ ] 实现分块推流：将长音频切成 ≤960 字节 Opus 帧逐帧下发
- [ ] 推送 `{"type": "state", "status": "speaking"}` 状态通知
- [ ] 补充测试

**分块推流时序：**
```
send state("speaking")
for chunk in opus_chunks:
    send binary(chunk)
    await asyncio.sleep(frame_duration)
send state("idle")
```

---

### Phase 5 — 生产加固

**目标：** 具备长期稳定运行的能力。

**任务：**
- [ ] 连接管理：最大并发数限制、心跳超时断开、异常重连
- [ ] 配置外置：`config.yaml` 或环境变量管理端口、模型选择、API 密钥等
- [ ] 结构化日志：JSON 格式日志，接入日志收集
- [ ] 指标监控：`/metrics` 端点暴露连接数、管线延迟、错误率
- [ ] systemd 服务单元文件，支持开机自启和自动重启
- [ ] HTTPS/WSS 支持（通过 nginx 反代或 uvicorn SSL 参数）
- [ ] HTTP 管理接口：`GET /sessions` 查看在线连接、`POST /broadcast` 广播消息

---

## 技术栈总览

| 层 | 当前（Phase 0） | 目标 |
|----|-----------------|------|
| 框架 | FastAPI + uvicorn | 不变 |
| WS 协议 | Starlette WebSocket | 不变 |
| 数据校验 | Pydantic v2 | 不变 |
| STT | mock | FunASR / SenseVoice / Whisper |
| LLM | mock (2s sleep) | OpenClaw agent |
| TTS | mock (静音帧) | edge-tts / CosyVoice |
| 音频编解码 | 无 | ffmpeg-python |
| 部署 | `python main.py` | systemd + nginx |

## 文件结构规划

```
openclaw-proxy/
├── main.py              # FastAPI 应用入口、WS 端点、帧路由
├── pipeline/
│   ├── __init__.py
│   ├── stt.py           # process_stt 实现
│   ├── llm.py           # call_openclaw 实现
│   └── tts.py           # process_tts 实现
├── models.py            # Pydantic 模型集中定义
├── session.py           # 连接会话状态管理、音频缓冲
├── config.py            # 配置加载（环境变量 / yaml）
├── tests/
│   ├── test_main.py
│   ├── test_stt.py
│   ├── test_llm.py
│   └── test_tts.py
├── requirements.txt
├── CLAUDE.md
└── PLAN.md
```

> 注：Phase 0 阶段全部逻辑集中在 `main.py`，从 Phase 1 开始按上述结构拆分模块。

-| LLM | mock (2s sleep) | OpenClaw agent |

+| LLM | `openclaw agent` CLI → Gateway :18789 | 同左（可选升级为 channel 插件） |


完成。最终验证一下代码和测试。


Bash

source venv/bin/activate && python -m pytest tests/ -v 2>&1

Final test run