手势识别改用纯OpenCV实现，无额外依赖

pyproject.toml

@@ -38,9 +38,6 @@ dependencies = [
     "supervision>=0.25.0",
     "huggingface_hub>=0.27.0",
     
-    # Gesture detection (MediaPipe - runs locally)
-    "mediapipe>=0.10.0",
-    
     # Sendspin synchronized audio (optional, for multi-room playback)
     "aiosendspin>=2.0.1",
 ]

reachy_mini_ha_voice/gesture_detector.py

@@ -1,19 +1,23 @@
-"""Gesture detection using MediaPipe Hands.
+"""Gesture detection using OpenCV skin color detection and contour analysis.
 
 Detects hand gestures for robot interaction:
-- Thumbs up: Confirmation/like
-- Open palm (stop): Stop speaking/cancel
+- thumbs_up: Confirmation/like
+- thumbs_down: Reject/dislike  
+- open_palm: Stop speaking/cancel (5 fingers)
+- fist: Pause/hold (0 fingers)
+- peace: Victory sign (2 fingers)
+- pointing_up: Attention (1 finger)
 
-Uses MediaPipe which is lightweight and can run alongside YOLO face detection.
+Uses pure OpenCV - no additional dependencies required.
 """
 
 from __future__ import annotations
 import logging
 from enum import Enum
-from typing import Optional, Tuple, Callable, List
-import threading
+from typing import Optional, Tuple, Callable
 import time
 
+import cv2
 import numpy as np
 from numpy.typing import NDArray
 
@@ -23,43 +27,30 @@ logger = logging.getLogger(__name__)
 class Gesture(Enum):
     """Recognized gestures."""
     NONE = "none"
-    THUMBS_UP = "thumbs_up"      # 👍 Confirm/like
-    THUMBS_DOWN = "thumbs_down"  # 👎 Reject/dislike
-    OPEN_PALM = "open_palm"      # ✋ Stop
-    FIST = "fist"                # ✊ Pause/hold
-    PEACE = "peace"              # ✌️ Victory/peace sign
-    OK = "ok"                    # 👌 OK sign
-    POINTING_UP = "pointing_up"  # ☝️ Attention/one
-    WAVE = "wave"                # 👋 Hello/goodbye (open palm moving)
+    THUMBS_UP = "thumbs_up"
+    THUMBS_DOWN = "thumbs_down"
+    OPEN_PALM = "open_palm"
+    FIST = "fist"
+    PEACE = "peace"
+    POINTING_UP = "pointing_up"
 
 
 class GestureDetector:
-    """Lightweight gesture detector using MediaPipe Hands.
-    
-    Designed to run alongside YOLO face detection with minimal overhead.
-    """
+    """Gesture detector using OpenCV skin detection and convex hull analysis."""
 
     def __init__(
         self,
-        min_detection_confidence: float = 0.6,
-        min_tracking_confidence: float = 0.5,
-        max_num_hands: int = 1,
+        min_hand_area: int = 5000,
+        max_hand_area: int = 150000,
     ) -> None:
         """Initialize gesture detector.
 
         Args:
-            min_detection_confidence: Minimum confidence for hand detection
-            min_tracking_confidence: Minimum confidence for hand tracking
-            max_num_hands: Maximum number of hands to detect (1 is faster)
+            min_hand_area: Minimum contour area to consider as hand
+            max_hand_area: Maximum contour area to consider as hand
         """
-        self._min_detection_confidence = min_detection_confidence
-        self._min_tracking_confidence = min_tracking_confidence
-        self._max_num_hands = max_num_hands
-        
-        self._hands = None
-        self._mp_hands = None
-        self._load_attempted = False
-        self._load_error: Optional[str] = None
+        self._min_hand_area = min_hand_area
+        self._max_hand_area = max_hand_area
         
         # Gesture callbacks
         self._on_thumbs_up: Optional[Callable[[], None]] = None
@@ -67,58 +58,28 @@ class GestureDetector:
         self._on_open_palm: Optional[Callable[[], None]] = None
         self._on_fist: Optional[Callable[[], None]] = None
         self._on_peace: Optional[Callable[[], None]] = None
-        self._on_ok: Optional[Callable[[], None]] = None
         self._on_pointing_up: Optional[Callable[[], None]] = None
         
-        # Gesture state (for debouncing)
+        # Gesture state
         self._last_gesture = Gesture.NONE
-        self._current_gesture = Gesture.NONE  # Currently active gesture (for HA entity)
+        self._current_gesture = Gesture.NONE
         self._gesture_start_time: Optional[float] = None
-        self._gesture_hold_threshold = 0.5  # Hold gesture for 0.5s to trigger
-        self._gesture_cooldown = 1.5  # Cooldown between triggers
+        self._gesture_hold_threshold = 0.5
+        self._gesture_cooldown = 1.5
         self._last_trigger_time: float = 0
-        self._gesture_clear_delay = 2.0  # Clear gesture after 2s of no detection
+        self._gesture_clear_delay = 2.0
         self._last_gesture_time: float = 0
         
-        # Load model
-        self._load_model()
-
-    def _load_model(self) -> None:
-        """Load MediaPipe Hands model."""
-        if self._load_attempted:
-            return
-        
-        self._load_attempted = True
-        
-        try:
-            import mediapipe as mp
-            
-            self._mp_hands = mp.solutions.hands
-            self._hands = self._mp_hands.Hands(
-                static_image_mode=False,
-                max_num_hands=self._max_num_hands,
-                min_detection_confidence=self._min_detection_confidence,
-                min_tracking_confidence=self._min_tracking_confidence,
-            )
-            logger.info("MediaPipe Hands loaded for gesture detection")
-        except ImportError as e:
-            self._load_error = f"Missing mediapipe: {e}"
-            logger.warning(
-                "Gesture detection disabled - missing mediapipe. "
-                "Install with: pip install mediapipe"
-            )
-        except Exception as e:
-            self._load_error = str(e)
-            logger.error("Failed to load MediaPipe Hands: %s", e)
+        logger.info("OpenCV gesture detector initialized")
 
     @property
     def is_available(self) -> bool:
-        """Check if gesture detector is available."""
-        return self._hands is not None
+        """Always available - uses OpenCV only."""
+        return True
 
     @property
     def current_gesture(self) -> Gesture:
-        """Get current detected gesture (for HA entity)."""
+        """Get current detected gesture."""
         return self._current_gesture
 
     def set_callbacks(
@@ -131,138 +92,158 @@ class GestureDetector:
         on_ok: Optional[Callable[[], None]] = None,
         on_pointing_up: Optional[Callable[[], None]] = None,
     ) -> None:
-        """Set gesture callbacks.
-        
-        Args:
-            on_thumbs_up: Called when thumbs up is detected (confirm/like)
-            on_thumbs_down: Called when thumbs down is detected (reject)
-            on_open_palm: Called when open palm is detected (stop)
-            on_fist: Called when fist is detected (pause)
-            on_peace: Called when peace sign is detected
-            on_ok: Called when OK sign is detected
-            on_pointing_up: Called when pointing up is detected
-        """
+        """Set gesture callbacks."""
         self._on_thumbs_up = on_thumbs_up
         self._on_thumbs_down = on_thumbs_down
         self._on_open_palm = on_open_palm
         self._on_fist = on_fist
         self._on_peace = on_peace
-        self._on_ok = on_ok
         self._on_pointing_up = on_pointing_up
 
-    def _get_landmark_coords(
-        self, landmarks, idx: int
-    ) -> Tuple[float, float, float]:
-        """Get x, y, z coordinates for a landmark."""
-        lm = landmarks.landmark[idx]
-        return lm.x, lm.y, lm.z
+    def _detect_skin(self, frame: NDArray[np.uint8]) -> NDArray[np.uint8]:
+        """Detect skin regions using YCrCb color space."""
+        # Convert to YCrCb
+        ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)
+        
+        # Skin color range in YCrCb
+        lower = np.array([0, 133, 77], dtype=np.uint8)
+        upper = np.array([255, 173, 127], dtype=np.uint8)
+        
+        # Create mask
+        mask = cv2.inRange(ycrcb, lower, upper)
+        
+        # Morphological operations to clean up
+        kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (5, 5))
+        mask = cv2.morphologyEx(mask, cv2.MORPH_OPEN, kernel)
+        mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
+        mask = cv2.GaussianBlur(mask, (5, 5), 0)
+        
+        return mask
+
+    def _find_hand_contour(self, mask: NDArray[np.uint8]) -> Optional[NDArray]:
+        """Find the largest hand contour in the mask."""
+        contours, _ = cv2.findContours(mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        
+        if not contours:
+            return None
+        
+        # Find largest contour within size bounds
+        valid_contours = []
+        for cnt in contours:
+            area = cv2.contourArea(cnt)
+            if self._min_hand_area < area < self._max_hand_area:
+                valid_contours.append((area, cnt))
+        
+        if not valid_contours:
+            return None
+        
+        # Return largest valid contour
+        valid_contours.sort(key=lambda x: x[0], reverse=True)
+        return valid_contours[0][1]
 
-    def _classify_gesture(self, hand_landmarks) -> Gesture:
-        """Classify hand gesture from landmarks.
-        
-        Landmark indices (MediaPipe):
-        - 0: Wrist
-        - 4: Thumb tip
-        - 8: Index finger tip
-        - 12: Middle finger tip
-        - 16: Ring finger tip
-        - 20: Pinky tip
-        - 2, 3: Thumb joints
-        - 5, 6, 7: Index joints
-        - etc.
+    def _count_fingers(self, contour: NDArray, frame_height: int) -> Tuple[int, bool, float]:
+        """Count extended fingers using convex hull defects.
+        
+        Returns:
+            Tuple of (finger_count, is_thumb_extended, hand_center_y_ratio)
         """
-        # Get key landmarks
-        wrist = self._get_landmark_coords(hand_landmarks, 0)
-        
-        # Thumb landmarks
-        thumb_tip = self._get_landmark_coords(hand_landmarks, 4)
-        thumb_ip = self._get_landmark_coords(hand_landmarks, 3)
-        thumb_mcp = self._get_landmark_coords(hand_landmarks, 2)
-        thumb_cmc = self._get_landmark_coords(hand_landmarks, 1)
-        
-        # Index finger landmarks
-        index_tip = self._get_landmark_coords(hand_landmarks, 8)
-        index_dip = self._get_landmark_coords(hand_landmarks, 7)
-        index_pip = self._get_landmark_coords(hand_landmarks, 6)
-        index_mcp = self._get_landmark_coords(hand_landmarks, 5)
-        
-        # Middle finger landmarks
-        middle_tip = self._get_landmark_coords(hand_landmarks, 12)
-        middle_dip = self._get_landmark_coords(hand_landmarks, 11)
-        middle_pip = self._get_landmark_coords(hand_landmarks, 10)
-        middle_mcp = self._get_landmark_coords(hand_landmarks, 9)
-        
-        # Ring finger landmarks
-        ring_tip = self._get_landmark_coords(hand_landmarks, 16)
-        ring_dip = self._get_landmark_coords(hand_landmarks, 15)
-        ring_pip = self._get_landmark_coords(hand_landmarks, 14)
-        ring_mcp = self._get_landmark_coords(hand_landmarks, 13)
-        
-        # Pinky landmarks
-        pinky_tip = self._get_landmark_coords(hand_landmarks, 20)
-        pinky_dip = self._get_landmark_coords(hand_landmarks, 19)
-        pinky_pip = self._get_landmark_coords(hand_landmarks, 18)
-        pinky_mcp = self._get_landmark_coords(hand_landmarks, 17)
-        
-        # Check if fingers are extended (tip above PIP in y, y increases downward)
-        index_extended = index_tip[1] < index_pip[1]
-        middle_extended = middle_tip[1] < middle_pip[1]
-        ring_extended = ring_tip[1] < ring_pip[1]
-        pinky_extended = pinky_tip[1] < pinky_pip[1]
-        
-        # Thumb extended check (horizontal distance from palm)
-        thumb_extended = abs(thumb_tip[0] - index_mcp[0]) > 0.08
-        
-        # Thumb pointing direction
-        thumb_pointing_up = thumb_tip[1] < thumb_mcp[1] - 0.05
-        thumb_pointing_down = thumb_tip[1] > thumb_mcp[1] + 0.05
-        
-        # Count extended fingers (excluding thumb)
-        extended_count = sum([index_extended, middle_extended, ring_extended, pinky_extended])
-        
-        # Fingers curled (tip below MCP)
-        index_curled = index_tip[1] > index_mcp[1]
-        middle_curled = middle_tip[1] > middle_mcp[1]
-        ring_curled = ring_tip[1] > ring_mcp[1]
-        pinky_curled = pinky_tip[1] > pinky_mcp[1]
-        all_fingers_curled = index_curled and middle_curled and ring_curled and pinky_curled
-        
-        # ===== Gesture Classification =====
-        
-        # 👍 Thumbs up: thumb pointing up, all other fingers curled
-        if thumb_pointing_up and thumb_extended and all_fingers_curled:
-            return Gesture.THUMBS_UP
-        
-        # 👎 Thumbs down: thumb pointing down, all other fingers curled
-        if thumb_pointing_down and thumb_extended and all_fingers_curled:
-            return Gesture.THUMBS_DOWN
-        
-        # ✊ Fist: all fingers curled including thumb tucked
-        if all_fingers_curled and not thumb_extended:
-            return Gesture.FIST
+        # Get convex hull
+        hull = cv2.convexHull(contour, returnPoints=False)
         
-        # ✌️ Peace: index and middle extended, others curled
-        if index_extended and middle_extended and not ring_extended and not pinky_extended:
-            return Gesture.PEACE
+        if len(hull) < 3:
+            return 0, False, 0.5
         
-        # ☝️ Pointing up: only index extended
-        if index_extended and not middle_extended and not ring_extended and not pinky_extended:
-            return Gesture.POINTING_UP
+        # Get convex hull points for centroid
+        hull_points = cv2.convexHull(contour)
         
-        # 👌 OK sign: thumb and index tips close together, other fingers extended
-        thumb_index_distance = self._distance(thumb_tip, index_tip)
-        if thumb_index_distance < 0.05 and middle_extended and ring_extended and pinky_extended:
-            return Gesture.OK
+        # Calculate centroid
+        M = cv2.moments(contour)
+        if M["m00"] == 0:
+            return 0, False, 0.5
         
-        # ✋ Open palm: all fingers extended
-        if extended_count >= 4:
+        cx = int(M["m10"] / M["m00"])
+        cy = int(M["m01"] / M["m00"])
+        center_y_ratio = cy / frame_height
+        
+        # Get bounding rect for reference
+        x, y, w, h = cv2.boundingRect(contour)
+        
+        # Get convexity defects
+        try:
+            defects = cv2.convexityDefects(contour, hull)
+        except cv2.error:
+            return 0, False, center_y_ratio
+        
+        if defects is None:
+            return 0, False, center_y_ratio
+        
+        # Count fingers based on defects
+        finger_count = 0
+        thumb_extended = False
+        
+        for i in range(defects.shape[0]):
+            s, e, f, d = defects[i, 0]
+            start = tuple(contour[s][0])
+            end = tuple(contour[e][0])
+            far = tuple(contour[f][0])
+            
+            # Calculate distances
+            a = np.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
+            b = np.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
+            c = np.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
+            
+            # Calculate angle using cosine rule
+            if b * c == 0:
+                continue
+            angle = np.arccos((b**2 + c**2 - a**2) / (2 * b * c))
+            
+            # Finger detected if angle < 90 degrees and defect depth is significant
+            if angle <= np.pi / 2 and d > 10000:
+                finger_count += 1
+                
+                # Check if this might be thumb (on side of hand)
+                if abs(start[0] - cx) > w * 0.3 or abs(end[0] - cx) > w * 0.3:
+                    thumb_extended = True
+        
+        # Add 1 because defects count spaces between fingers
+        if finger_count > 0:
+            finger_count += 1
+        
+        return min(finger_count, 5), thumb_extended, center_y_ratio
+
+    def _classify_gesture(self, contour: NDArray, frame_height: int) -> Gesture:
+        """Classify gesture based on contour analysis."""
+        finger_count, thumb_extended, center_y = self._count_fingers(contour, frame_height)
+        
+        # Get contour properties
+        x, y, w, h = cv2.boundingRect(contour)
+        aspect_ratio = float(w) / h if h > 0 else 1.0
+        
+        # Get hull area ratio (solidity)
+        hull = cv2.convexHull(contour)
+        hull_area = cv2.contourArea(hull)
+        contour_area = cv2.contourArea(contour)
+        solidity = contour_area / hull_area if hull_area > 0 else 0
+        
+        # Classify based on finger count and shape
+        if finger_count >= 4:
             return Gesture.OPEN_PALM
+        elif finger_count == 2:
+            return Gesture.PEACE
+        elif finger_count == 1:
+            # Check if pointing up or thumb gesture
+            if aspect_ratio < 0.7:  # Tall and narrow = pointing up
+                return Gesture.POINTING_UP
+            elif thumb_extended:
+                # Check thumb direction based on position
+                if center_y < 0.5:  # Hand in upper half
+                    return Gesture.THUMBS_UP
+                else:
+                    return Gesture.THUMBS_DOWN
+        elif finger_count == 0 and solidity > 0.8:
+            return Gesture.FIST
         
         return Gesture.NONE
-    
-    def _distance(self, p1: Tuple[float, float, float], p2: Tuple[float, float, float]) -> float:
-        """Calculate Euclidean distance between two points."""
-        return ((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2 + (p1[2] - p2[2])**2) ** 0.5
 
     def detect(self, frame: NDArray[np.uint8]) -> Gesture:
         """Detect gesture in frame.
@@ -271,44 +252,27 @@ class GestureDetector:
             frame: BGR image from camera
             
         Returns:
-            Detected gesture (may be NONE)
+            Detected gesture
         """
-        if not self.is_available:
-            return Gesture.NONE
-        
         try:
-            import cv2
-            
-            # Convert BGR to RGB for MediaPipe
-            rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+            # Detect skin
+            mask = self._detect_skin(frame)
             
-            # Process frame
-            results = self._hands.process(rgb_frame)
+            # Find hand contour
+            contour = self._find_hand_contour(mask)
             
-            if not results.multi_hand_landmarks:
+            if contour is None:
                 return Gesture.NONE
             
-            # Use first detected hand
-            hand_landmarks = results.multi_hand_landmarks[0]
-            gesture = self._classify_gesture(hand_landmarks)
-            
-            return gesture
+            # Classify gesture
+            return self._classify_gesture(contour, frame.shape[0])
             
         except Exception as e:
             logger.debug("Gesture detection error: %s", e)
             return Gesture.NONE
 
     def process_frame(self, frame: NDArray[np.uint8]) -> Optional[Gesture]:
-        """Process frame and trigger callbacks if gesture held.
-        
-        This method handles debouncing and cooldown logic.
-        
-        Args:
-            frame: BGR image from camera
-            
-        Returns:
-            Triggered gesture or None
-        """
+        """Process frame and trigger callbacks if gesture held."""
         current_gesture = self.detect(frame)
         current_time = time.time()
         
@@ -317,7 +281,6 @@ class GestureDetector:
             self._current_gesture = current_gesture
             self._last_gesture_time = current_time
         elif current_time - self._last_gesture_time > self._gesture_clear_delay:
-            # Clear gesture after delay
             self._current_gesture = Gesture.NONE
         
         # Check cooldown
@@ -335,37 +298,32 @@ class GestureDetector:
             hold_duration = current_time - self._gesture_start_time
             
             if hold_duration >= self._gesture_hold_threshold:
-                # Trigger callback
                 self._last_trigger_time = current_time
-                self._gesture_start_time = None  # Reset to prevent re-trigger
+                self._gesture_start_time = None
                 
-                # Get callback for this gesture
                 callback = self._get_callback_for_gesture(current_gesture)
                 if callback:
                     logger.info("Gesture triggered: %s", current_gesture.value)
                     try:
                         callback()
                     except Exception as e:
-                        logger.error("Gesture callback error (%s): %s", current_gesture.value, e)
+                        logger.error("Gesture callback error: %s", e)
                     return current_gesture
         
         return None
-    
+
     def _get_callback_for_gesture(self, gesture: Gesture) -> Optional[Callable[[], None]]:
-        """Get the callback function for a gesture."""
+        """Get callback for gesture."""
         callbacks = {
             Gesture.THUMBS_UP: self._on_thumbs_up,
             Gesture.THUMBS_DOWN: self._on_thumbs_down,
             Gesture.OPEN_PALM: self._on_open_palm,
             Gesture.FIST: self._on_fist,
             Gesture.PEACE: self._on_peace,
-            Gesture.OK: self._on_ok,
             Gesture.POINTING_UP: self._on_pointing_up,
         }
         return callbacks.get(gesture)
 
     def close(self) -> None:
         """Release resources."""
-        if self._hands is not None:
-            self._hands.close()
-            self._hands = None
+        pass