"feat-camera-add-mjpeg-streaming-server-for-home-assistant"

.gitignore

@@ -73,3 +73,4 @@ models/
 reachy_mini/
 reachy_mini_conversation_app/
 linux-voice-assistant/
+reachy-mini-desktop-app/

PROJECT_PLAN.md

@@ -8,6 +8,7 @@
 1. [linux-voice-assistant](linux-voice-assistant)
 2. [Reachy Mini SDK](reachy_mini)
 3. [reachy_mini_conversation_app](reachy_mini_conversation_app)
+4. [reachy-mini-desktop-app](reachy-mini-desktop-app)
 
 ## 核心设计原则
 

README.md

@@ -19,6 +19,7 @@ A voice assistant application for **Reachy Mini robot** that integrates with Hom
 
 - **Local Wake Word Detection**: Uses microWakeWord for offline wake word detection
 - **ESPHome Integration**: Seamlessly connects to Home Assistant
+- **Camera Streaming**: MJPEG video stream for Home Assistant Generic Camera integration
 - **Motion Control**: Head movements and antenna animations during voice interaction
 - **Zero Configuration**: Install and run - all settings are managed in Home Assistant
 - **Full Robot Control**: Expose 30+ entities to Home Assistant for complete robot control
@@ -55,6 +56,20 @@ The app runs automatically when installed on Reachy Mini. After installation:
 4. Enter your Reachy Mini's IP address with port `6053`
 5. The voice assistant will be automatically discovered
 
+### Camera Setup
+
+The camera stream is available at `http://<reachy-mini-ip>:8081/stream`. To add it to Home Assistant:
+
+1. Go to **Settings** -> **Devices & Services** -> **Add Integration**
+2. Search for **Generic Camera**
+3. Enter the stream URL: `http://<reachy-mini-ip>:8081/stream`
+4. Set content type to `image/jpeg`
+
+You can also access:
+- **Live Stream**: `http://<reachy-mini-ip>:8081/stream` - MJPEG video stream
+- **Snapshot**: `http://<reachy-mini-ip>:8081/snapshot` - Single JPEG image
+- **Status Page**: `http://<reachy-mini-ip>:8081/` - Web interface with stream preview
+
 ### Wake Words
 
 Default wake word: **"Okay Nabu"**
@@ -115,11 +130,14 @@ This application exposes 30+ entities to Home Assistant for complete robot contr
         |
         v
 [Head Motion & Antenna Animation]
+
+[Reachy Mini Camera] -> [MJPEG Server :8081] -> [Home Assistant Generic Camera]
 ```
 
 - **Wake word detection** runs locally on Reachy Mini
 - **Speech-to-Text (STT)** and **Text-to-Speech (TTS)** are handled by Home Assistant
 - **Motion feedback** provides visual response during voice interaction
+- **Camera streaming** provides real-time video feed to Home Assistant
 
 ## Project Structure
 
@@ -129,6 +147,7 @@ reachy_mini_ha_voice/
 │   ├── __init__.py
 │   ├── main.py              # App entry point
 │   ├── voice_assistant.py   # Voice assistant service
+│   ├── camera_server.py     # MJPEG camera streaming server
 │   ├── satellite.py         # ESPHome protocol handler
 │   ├── audio_player.py      # Audio playback
 │   ├── motion.py            # Motion control
@@ -149,6 +168,7 @@ reachy_mini_ha_voice/
 - `reachy-mini` - Reachy Mini SDK
 - `aioesphomeapi` - ESPHome protocol
 - `pymicro-wakeword` - Wake word detection
+- `opencv-python` - Camera streaming
 - `sounddevice` / `soundfile` - Audio processing
 - `zeroconf` - mDNS discovery
 

pyproject.toml

@@ -18,6 +18,9 @@ dependencies = [
     "soundfile>=0.12.0",
     "numpy>=1.24.0",
 
+    # Camera streaming
+    "opencv-python>=4.8.0",
+
     # Wake word detection (local)
     # STT/TTS is handled by Home Assistant, not locally
     "pymicro-wakeword>=2.0.0,<3.0.0",

reachy_mini_ha_voice/camera_server.py

@@ -0,0 +1,365 @@
+"""
+MJPEG Camera Server for Reachy Mini.
+
+This module provides an HTTP server that streams camera frames from Reachy Mini
+as MJPEG, which can be integrated with Home Assistant via Generic Camera.
+"""
+
+import asyncio
+import logging
+import threading
+import time
+from typing import Optional, TYPE_CHECKING
+
+import cv2
+import numpy as np
+
+if TYPE_CHECKING:
+    from reachy_mini import ReachyMini
+
+_LOGGER = logging.getLogger(__name__)
+
+# MJPEG boundary string
+MJPEG_BOUNDARY = "frame"
+
+
+class MJPEGCameraServer:
+    """
+    MJPEG streaming server for Reachy Mini camera.
+
+    Provides HTTP endpoints:
+    - /stream - MJPEG video stream
+    - /snapshot - Single JPEG image
+    - / - Simple status page
+    """
+
+    def __init__(
+        self,
+        reachy_mini: Optional["ReachyMini"] = None,
+        host: str = "0.0.0.0",
+        port: int = 8081,
+        fps: int = 15,
+        quality: int = 80,
+    ):
+        """
+        Initialize the MJPEG camera server.
+
+        Args:
+            reachy_mini: Reachy Mini robot instance (can be None for testing)
+            host: Host address to bind to
+            port: Port number for the HTTP server
+            fps: Target frames per second for the stream
+            quality: JPEG quality (1-100)
+        """
+        self.reachy_mini = reachy_mini
+        self.host = host
+        self.port = port
+        self.fps = fps
+        self.quality = quality
+
+        self._server: Optional[asyncio.Server] = None
+        self._running = False
+        self._frame_interval = 1.0 / fps
+        self._last_frame: Optional[bytes] = None
+        self._last_frame_time: float = 0
+        self._frame_lock = threading.Lock()
+
+        # Frame capture thread
+        self._capture_thread: Optional[threading.Thread] = None
+
+    async def start(self) -> None:
+        """Start the MJPEG camera server."""
+        if self._running:
+            _LOGGER.warning("Camera server already running")
+            return
+
+        self._running = True
+
+        # Start frame capture thread
+        self._capture_thread = threading.Thread(
+            target=self._capture_frames,
+            daemon=True,
+            name="camera-capture"
+        )
+        self._capture_thread.start()
+
+        # Start HTTP server
+        self._server = await asyncio.start_server(
+            self._handle_client,
+            self.host,
+            self.port,
+        )
+
+        _LOGGER.info("MJPEG Camera server started on http://%s:%d", self.host, self.port)
+        _LOGGER.info("  Stream URL: http://<ip>:%d/stream", self.port)
+        _LOGGER.info("  Snapshot URL: http://<ip>:%d/snapshot", self.port)
+
+    async def stop(self) -> None:
+        """Stop the MJPEG camera server."""
+        self._running = False
+
+        if self._capture_thread:
+            self._capture_thread.join(timeout=2.0)
+            self._capture_thread = None
+
+        if self._server:
+            self._server.close()
+            await self._server.wait_closed()
+            self._server = None
+
+        _LOGGER.info("MJPEG Camera server stopped")
+
+    def _capture_frames(self) -> None:
+        """Background thread to capture frames from Reachy Mini."""
+        _LOGGER.info("Starting camera capture thread")
+
+        while self._running:
+            try:
+                frame = self._get_camera_frame()
+
+                if frame is not None:
+                    # Encode frame as JPEG
+                    encode_params = [cv2.IMWRITE_JPEG_QUALITY, self.quality]
+                    success, jpeg_data = cv2.imencode('.jpg', frame, encode_params)
+
+                    if success:
+                        with self._frame_lock:
+                            self._last_frame = jpeg_data.tobytes()
+                            self._last_frame_time = time.time()
+
+                # Sleep to maintain target FPS
+                time.sleep(self._frame_interval)
+
+            except Exception as e:
+                _LOGGER.error("Error capturing frame: %s", e)
+                time.sleep(0.5)
+
+        _LOGGER.info("Camera capture thread stopped")
+
+    def _get_camera_frame(self) -> Optional[np.ndarray]:
+        """Get a frame from Reachy Mini's camera."""
+        if self.reachy_mini is None:
+            # Return a test pattern if no robot connected
+            return self._generate_test_frame()
+
+        try:
+            frame = self.reachy_mini.media.get_frame()
+            return frame
+        except Exception as e:
+            _LOGGER.debug("Failed to get camera frame: %s", e)
+            return None
+
+    def _generate_test_frame(self) -> np.ndarray:
+        """Generate a test pattern frame when no camera is available."""
+        # Create a simple test pattern
+        frame = np.zeros((480, 640, 3), dtype=np.uint8)
+
+        # Add some visual elements
+        cv2.putText(
+            frame,
+            "Reachy Mini Camera",
+            (150, 200),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            1.2,
+            (255, 255, 255),
+            2,
+        )
+        cv2.putText(
+            frame,
+            "No camera connected",
+            (180, 280),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.8,
+            (128, 128, 128),
+            1,
+        )
+
+        # Add timestamp
+        timestamp = time.strftime("%Y-%m-%d %H:%M:%S")
+        cv2.putText(
+            frame,
+            timestamp,
+            (220, 350),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.6,
+            (0, 255, 0),
+            1,
+        )
+
+        return frame
+
+    def get_snapshot(self) -> Optional[bytes]:
+        """Get the latest frame as JPEG bytes."""
+        with self._frame_lock:
+            return self._last_frame
+
+    async def _handle_client(
+        self,
+        reader: asyncio.StreamReader,
+        writer: asyncio.StreamWriter,
+    ) -> None:
+        """Handle incoming HTTP client connections."""
+        try:
+            # Read HTTP request
+            request_line = await asyncio.wait_for(
+                reader.readline(),
+                timeout=10.0
+            )
+            request = request_line.decode('utf-8', errors='ignore').strip()
+
+            # Read headers (we don't need them but must consume them)
+            while True:
+                line = await asyncio.wait_for(reader.readline(), timeout=5.0)
+                if line == b'\r\n' or line == b'\n' or line == b'':
+                    break
+
+            # Parse request path
+            parts = request.split(' ')
+            if len(parts) >= 2:
+                path = parts[1]
+            else:
+                path = '/'
+
+            _LOGGER.debug("HTTP request: %s", request)
+
+            if path == '/stream':
+                await self._handle_stream(writer)
+            elif path == '/snapshot':
+                await self._handle_snapshot(writer)
+            else:
+                await self._handle_index(writer)
+
+        except asyncio.TimeoutError:
+            _LOGGER.debug("Client connection timeout")
+        except ConnectionResetError:
+            _LOGGER.debug("Client connection reset")
+        except Exception as e:
+            _LOGGER.error("Error handling client: %s", e)
+        finally:
+            try:
+                writer.close()
+                await writer.wait_closed()
+            except Exception:
+                pass
+
+    async def _handle_index(self, writer: asyncio.StreamWriter) -> None:
+        """Handle index page request."""
+        html = f"""<!DOCTYPE html>
+<html>
+<head>
+    <title>Reachy Mini Camera</title>
+    <style>
+        body {{ font-family: Arial, sans-serif; margin: 40px; background: #1a1a2e; color: #eee; }}
+        h1 {{ color: #00d4ff; }}
+        .container {{ max-width: 800px; margin: 0 auto; }}
+        .stream {{ width: 100%; max-width: 640px; border: 2px solid #00d4ff; border-radius: 8px; }}
+        a {{ color: #00d4ff; }}
+        .info {{ background: #16213e; padding: 20px; border-radius: 8px; margin-top: 20px; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>Reachy Mini Camera</h1>
+        <img class="stream" src="/stream" alt="Camera Stream">
+        <div class="info">
+            <h3>Endpoints:</h3>
+            <ul>
+                <li><a href="/stream">/stream</a> - MJPEG video stream</li>
+                <li><a href="/snapshot">/snapshot</a> - Single JPEG snapshot</li>
+            </ul>
+            <h3>Home Assistant Integration:</h3>
+            <p>Add a Generic Camera with URL: <code>http://&lt;ip&gt;:{self.port}/stream</code></p>
+        </div>
+    </div>
+</body>
+</html>"""
+
+        response = (
+            "HTTP/1.1 200 OK\r\n"
+            "Content-Type: text/html; charset=utf-8\r\n"
+            f"Content-Length: {len(html)}\r\n"
+            "Connection: close\r\n"
+            "\r\n"
+        )
+
+        writer.write(response.encode('utf-8'))
+        writer.write(html.encode('utf-8'))
+        await writer.drain()
+
+    async def _handle_snapshot(self, writer: asyncio.StreamWriter) -> None:
+        """Handle snapshot request - return single JPEG image."""
+        jpeg_data = self.get_snapshot()
+
+        if jpeg_data is None:
+            response = (
+                "HTTP/1.1 503 Service Unavailable\r\n"
+                "Content-Type: text/plain\r\n"
+                "Connection: close\r\n"
+                "\r\n"
+                "No frame available"
+            )
+            writer.write(response.encode('utf-8'))
+        else:
+            response = (
+                "HTTP/1.1 200 OK\r\n"
+                "Content-Type: image/jpeg\r\n"
+                f"Content-Length: {len(jpeg_data)}\r\n"
+                "Cache-Control: no-cache, no-store, must-revalidate\r\n"
+                "Connection: close\r\n"
+                "\r\n"
+            )
+            writer.write(response.encode('utf-8'))
+            writer.write(jpeg_data)
+
+        await writer.drain()
+
+    async def _handle_stream(self, writer: asyncio.StreamWriter) -> None:
+        """Handle MJPEG stream request."""
+        # Send MJPEG headers
+        response = (
+            "HTTP/1.1 200 OK\r\n"
+            f"Content-Type: multipart/x-mixed-replace; boundary={MJPEG_BOUNDARY}\r\n"
+            "Cache-Control: no-cache, no-store, must-revalidate\r\n"
+            "Connection: keep-alive\r\n"
+            "\r\n"
+        )
+        writer.write(response.encode('utf-8'))
+        await writer.drain()
+
+        _LOGGER.debug("Started MJPEG stream")
+
+        last_sent_time = 0
+
+        try:
+            while self._running:
+                # Get latest frame
+                with self._frame_lock:
+                    jpeg_data = self._last_frame
+                    frame_time = self._last_frame_time
+
+                # Only send if we have a new frame
+                if jpeg_data is not None and frame_time > last_sent_time:
+                    # Send MJPEG frame
+                    frame_header = (
+                        f"--{MJPEG_BOUNDARY}\r\n"
+                        "Content-Type: image/jpeg\r\n"
+                        f"Content-Length: {len(jpeg_data)}\r\n"
+                        "\r\n"
+                    )
+
+                    writer.write(frame_header.encode('utf-8'))
+                    writer.write(jpeg_data)
+                    writer.write(b"\r\n")
+                    await writer.drain()
+
+                    last_sent_time = frame_time
+
+                # Small delay to prevent busy loop
+                await asyncio.sleep(0.01)
+
+        except (ConnectionResetError, BrokenPipeError):
+            _LOGGER.debug("Client disconnected from stream")
+        except Exception as e:
+            _LOGGER.error("Error in MJPEG stream: %s", e)
+
+        _LOGGER.debug("Ended MJPEG stream")

reachy_mini_ha_voice/main.py

@@ -132,15 +132,19 @@ class ReachyMiniHAVoiceApp(ReachyMiniApp):
             logger.info("Home Assistant Voice Assistant Started!")
             logger.info("=" * 50)
             logger.info("ESPHome Server: 0.0.0.0:6053")
+            logger.info("Camera Server: 0.0.0.0:8081")
             logger.info("Wake word: Okay Nabu")
             if reachy_mini:
                 logger.info("Motion control: enabled")
+                logger.info("Camera: enabled (Reachy Mini)")
             else:
                 logger.info("Motion control: disabled (no robot)")
+                logger.info("Camera: test pattern (no robot)")
             logger.info("=" * 50)
             logger.info("To connect from Home Assistant:")
             logger.info("  Settings -> Devices & Services -> Add Integration")
             logger.info("  -> ESPHome -> Enter this device's IP:6053")
+            logger.info("  -> Generic Camera -> http://<ip>:8081/stream")
             logger.info("=" * 50)
 
             # Wait for stop signal

reachy_mini_ha_voice/voice_assistant.py

@@ -24,6 +24,7 @@ from .satellite import VoiceSatelliteProtocol
 from .util import get_mac
 from .zeroconf import HomeAssistantZeroconf
 from .motion import ReachyMiniMotion
+from .camera_server import MJPEGCameraServer
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -43,12 +44,16 @@ class VoiceAssistantService:
         host: str = "0.0.0.0",
         port: int = 6053,
         wake_model: str = "okay_nabu",
+        camera_port: int = 8081,
+        camera_enabled: bool = True,
     ):
         self.reachy_mini = reachy_mini
         self.name = name
         self.host = host
         self.port = port
         self.wake_model = wake_model
+        self.camera_port = camera_port
+        self.camera_enabled = camera_enabled
 
         self._server = None
         self._discovery = None
@@ -56,6 +61,7 @@ class VoiceAssistantService:
         self._running = False
         self._state: Optional[ServerState] = None
         self._motion = ReachyMiniMotion(reachy_mini)
+        self._camera_server: Optional[MJPEGCameraServer] = None
 
     async def start(self) -> None:
         """Start the voice assistant service."""
@@ -143,6 +149,17 @@ class VoiceAssistantService:
         self._discovery = HomeAssistantZeroconf(port=self.port, name=self.name)
         await self._discovery.register_server()
 
+        # Start camera server if enabled
+        if self.camera_enabled:
+            self._camera_server = MJPEGCameraServer(
+                reachy_mini=self.reachy_mini,
+                host=self.host,
+                port=self.camera_port,
+                fps=15,
+                quality=80,
+            )
+            await self._camera_server.start()
+
         _LOGGER.info("Voice assistant service started on %s:%s", self.host, self.port)
 
     async def stop(self) -> None:
@@ -161,6 +178,11 @@ class VoiceAssistantService:
         if self._discovery:
             await self._discovery.unregister_server()
 
+        # Stop camera server
+        if self._camera_server:
+            await self._camera_server.stop()
+            self._camera_server = None
+
         # Stop Reachy Mini media system
         if self.reachy_mini is not None:
             try: