refactor: comprehensive code quality, security, and release readiness improvements
Lint & Test / test (push) Failing after 48s
Lint & Test / test (push) Failing after 48s
Security: tighten CORS defaults, add webhook rate limiting, fix XSS in automations, guard WebSocket JSON.parse, validate ADB address input, seal debug exception leak, URL-encode WS tokens, CSS.escape in selectors. Code quality: add Pydantic models for brightness/power endpoints, fix thread safety and name uniqueness in DeviceStore, immutable update pattern, split 6 oversized files into 16 focused modules, enable TypeScript strictNullChecks (741→102 errors), type state variables, add dom-utils helper, migrate 3 modules from inline onclick to event delegation, ProcessorDependencies dataclass. Performance: async store saves, health endpoint log level, command palette debounce, optimized entity-events comparison, fix service worker precache list. Testing: expand from 45 to 293 passing tests — add store tests (141), route tests (25), core logic tests (42), E2E flow tests (33), organize into tests/api/, tests/storage/, tests/core/, tests/e2e/. DevOps: CI test pipeline, pre-commit config, Dockerfile multi-stage build with non-root user and health check, docker-compose improvements, version bump to 0.2.0. Docs: rewrite CLAUDE.md (202→56 lines), server/CLAUDE.md (212→76), create contexts/server-operations.md, fix .js→.ts references, fix env var prefix in README, rewrite INSTALLATION.md, add CONTRIBUTING.md and .env.example.
This commit is contained in:
@@ -1,6 +1,10 @@
|
||||
"""Target processing pipeline."""
|
||||
|
||||
from wled_controller.core.processing.processor_manager import ProcessorManager
|
||||
from wled_controller.core.processing.processor_manager import (
|
||||
DeviceState,
|
||||
ProcessorDependencies,
|
||||
ProcessorManager,
|
||||
)
|
||||
from wled_controller.core.processing.target_processor import (
|
||||
DeviceInfo,
|
||||
ProcessingMetrics,
|
||||
@@ -10,7 +14,9 @@ from wled_controller.core.processing.target_processor import (
|
||||
|
||||
__all__ = [
|
||||
"DeviceInfo",
|
||||
"DeviceState",
|
||||
"ProcessingMetrics",
|
||||
"ProcessorDependencies",
|
||||
"ProcessorManager",
|
||||
"TargetContext",
|
||||
"TargetProcessor",
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Auto-restart mixin for ProcessorManager.
|
||||
|
||||
Handles crash detection, exponential backoff, and automatic restart
|
||||
of failed target processors.
|
||||
|
||||
Extracted from processor_manager.py to keep files under 800 lines.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Dict, Optional
|
||||
|
||||
from wled_controller.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# Auto-restart constants
|
||||
RESTART_MAX_ATTEMPTS = 5 # max restarts within the window
|
||||
RESTART_WINDOW_SEC = 300 # 5 minutes — reset counter after stable period
|
||||
RESTART_BACKOFF_BASE = 2.0 # initial backoff seconds
|
||||
RESTART_BACKOFF_MAX = 30.0 # cap backoff at 30s
|
||||
|
||||
|
||||
@dataclass
|
||||
class RestartState:
|
||||
"""Per-target auto-restart tracking."""
|
||||
attempts: int = 0
|
||||
first_crash_time: float = 0.0
|
||||
last_crash_time: float = 0.0
|
||||
restart_task: Optional[asyncio.Task] = None
|
||||
enabled: bool = True # disabled on manual stop
|
||||
|
||||
|
||||
class AutoRestartMixin:
|
||||
"""Mixin providing auto-restart logic for crashed target processors.
|
||||
|
||||
Requires the host class to have:
|
||||
_processors: Dict[str, TargetProcessor]
|
||||
_restart_states: Dict[str, RestartState]
|
||||
fire_event(event: dict) -> None
|
||||
start_processing(target_id: str) -> coroutine
|
||||
"""
|
||||
|
||||
def _on_task_done(self, target_id: str, task: asyncio.Task) -> None:
|
||||
"""Task done callback — detects crashes and schedules auto-restart."""
|
||||
# Ignore graceful cancellation (manual stop)
|
||||
if task.cancelled():
|
||||
return
|
||||
|
||||
exc = task.exception()
|
||||
if exc is None:
|
||||
return # Clean exit (shouldn't happen, but harmless)
|
||||
|
||||
rs = self._restart_states.get(target_id)
|
||||
if not rs or not rs.enabled:
|
||||
return # Auto-restart disabled (manual stop was called)
|
||||
|
||||
now = time.monotonic()
|
||||
|
||||
# Reset counter if previous crash window expired
|
||||
if rs.first_crash_time and (now - rs.first_crash_time) > RESTART_WINDOW_SEC:
|
||||
rs.attempts = 0
|
||||
rs.first_crash_time = 0.0
|
||||
|
||||
rs.attempts += 1
|
||||
rs.last_crash_time = now
|
||||
if not rs.first_crash_time:
|
||||
rs.first_crash_time = now
|
||||
|
||||
if rs.attempts > RESTART_MAX_ATTEMPTS:
|
||||
logger.error(
|
||||
f"[AUTO-RESTART] Target {target_id} crashed {rs.attempts} times "
|
||||
f"in {now - rs.first_crash_time:.0f}s — giving up"
|
||||
)
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_exhausted": True,
|
||||
})
|
||||
return
|
||||
|
||||
backoff = min(
|
||||
RESTART_BACKOFF_BASE * (2 ** (rs.attempts - 1)),
|
||||
RESTART_BACKOFF_MAX,
|
||||
)
|
||||
logger.warning(
|
||||
f"[AUTO-RESTART] Target {target_id} crashed (attempt {rs.attempts}/"
|
||||
f"{RESTART_MAX_ATTEMPTS}), restarting in {backoff:.1f}s"
|
||||
)
|
||||
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_in": backoff,
|
||||
"auto_restart_attempt": rs.attempts,
|
||||
})
|
||||
|
||||
# Schedule the restart (runs in the event loop)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
logger.error(f"[AUTO-RESTART] No running event loop for {target_id}")
|
||||
return
|
||||
|
||||
rs.restart_task = loop.create_task(self._auto_restart(target_id, backoff))
|
||||
|
||||
async def _auto_restart(self, target_id: str, delay: float) -> None:
|
||||
"""Wait for backoff delay, then restart the target processor."""
|
||||
try:
|
||||
await asyncio.sleep(delay)
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"[AUTO-RESTART] Restart cancelled for {target_id}")
|
||||
return
|
||||
|
||||
rs = self._restart_states.get(target_id)
|
||||
if not rs or not rs.enabled:
|
||||
logger.info(f"[AUTO-RESTART] Restart aborted for {target_id} (disabled)")
|
||||
return
|
||||
|
||||
proc = self._processors.get(target_id)
|
||||
if proc is None:
|
||||
logger.warning(f"[AUTO-RESTART] Target {target_id} no longer registered")
|
||||
return
|
||||
if proc.is_running:
|
||||
logger.info(f"[AUTO-RESTART] Target {target_id} already running, skipping")
|
||||
return
|
||||
|
||||
logger.info(f"[AUTO-RESTART] Restarting target {target_id} (attempt {rs.attempts})")
|
||||
try:
|
||||
await self.start_processing(target_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[AUTO-RESTART] Failed to restart {target_id}: {e}")
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_error": str(e),
|
||||
})
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Device health monitoring mixin for ProcessorManager.
|
||||
|
||||
Extracted from processor_manager.py to keep files under 800 lines.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from wled_controller.core.devices.led_client import (
|
||||
DeviceHealth,
|
||||
check_device_health,
|
||||
get_device_capabilities,
|
||||
)
|
||||
from wled_controller.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class DeviceHealthMixin:
|
||||
"""Mixin providing health monitoring loop and health check methods.
|
||||
|
||||
Requires the host class to have:
|
||||
_devices: Dict[str, DeviceState]
|
||||
_processors: Dict[str, TargetProcessor]
|
||||
_health_monitoring_active: bool
|
||||
_http_client: Optional[httpx.AsyncClient]
|
||||
_device_store: object
|
||||
fire_event(event: dict) -> None
|
||||
_get_http_client() -> httpx.AsyncClient
|
||||
"""
|
||||
|
||||
# ===== HEALTH MONITORING =====
|
||||
|
||||
async def start_health_monitoring(self):
|
||||
"""Start background health checks for all registered devices."""
|
||||
self._health_monitoring_active = True
|
||||
for device_id in self._devices:
|
||||
self._start_device_health_check(device_id)
|
||||
await self._metrics_history.start()
|
||||
logger.info("Started health monitoring for all devices")
|
||||
|
||||
async def stop_health_monitoring(self):
|
||||
"""Stop all background health checks."""
|
||||
self._health_monitoring_active = False
|
||||
for device_id in list(self._devices.keys()):
|
||||
self._stop_device_health_check(device_id)
|
||||
logger.info("Stopped health monitoring for all devices")
|
||||
|
||||
def _start_device_health_check(self, device_id: str):
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
# Skip periodic health checks for virtual devices (always online)
|
||||
if "health_check" not in get_device_capabilities(state.device_type):
|
||||
state.health = DeviceHealth(
|
||||
online=True, latency_ms=0.0, last_checked=datetime.now(timezone.utc)
|
||||
)
|
||||
return
|
||||
if state.health_task and not state.health_task.done():
|
||||
return
|
||||
state.health_task = asyncio.create_task(self._health_check_loop(device_id))
|
||||
|
||||
def _stop_device_health_check(self, device_id: str):
|
||||
state = self._devices.get(device_id)
|
||||
if not state or not state.health_task:
|
||||
return
|
||||
state.health_task.cancel()
|
||||
state.health_task = None
|
||||
|
||||
def _device_is_processing(self, device_id: str) -> bool:
|
||||
"""Check if any target is actively streaming to this device."""
|
||||
return any(
|
||||
p.device_id == device_id and p.is_running
|
||||
for p in self._processors.values()
|
||||
)
|
||||
|
||||
def _is_device_streaming(self, device_id: str) -> bool:
|
||||
"""Check if any running processor targets this device."""
|
||||
for proc in self._processors.values():
|
||||
if getattr(proc, 'device_id', None) == device_id and proc.is_running:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _health_check_loop(self, device_id: str):
|
||||
"""Background loop that periodically checks a device.
|
||||
|
||||
Uses adaptive intervals: 10s for actively streaming devices,
|
||||
60s for idle devices, to balance responsiveness with overhead.
|
||||
"""
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
|
||||
ACTIVE_INTERVAL = 10 # streaming devices — faster detection
|
||||
IDLE_INTERVAL = 60 # idle devices — less overhead
|
||||
|
||||
try:
|
||||
while self._health_monitoring_active:
|
||||
await self._check_device_health(device_id)
|
||||
interval = ACTIVE_INTERVAL if self._is_device_streaming(device_id) else IDLE_INTERVAL
|
||||
await asyncio.sleep(interval)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Fatal error in health check loop for {device_id}: {e}")
|
||||
|
||||
async def _check_device_health(self, device_id: str):
|
||||
"""Check device health. Also auto-syncs LED count if changed."""
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
prev_online = state.health.online
|
||||
client = await self._get_http_client()
|
||||
state.health = await check_device_health(
|
||||
state.device_type, state.device_url, client, state.health,
|
||||
)
|
||||
|
||||
# Fire event when online status changes
|
||||
if state.health.online != prev_online:
|
||||
self.fire_event({
|
||||
"type": "device_health_changed",
|
||||
"device_id": device_id,
|
||||
"online": state.health.online,
|
||||
"latency_ms": state.health.latency_ms,
|
||||
})
|
||||
|
||||
# Auto-sync LED count
|
||||
reported = state.health.device_led_count
|
||||
if reported and reported != state.led_count and self._device_store:
|
||||
old_count = state.led_count
|
||||
logger.info(
|
||||
f"Device {device_id} LED count changed: {old_count} → {reported}"
|
||||
)
|
||||
try:
|
||||
self._device_store.update_device(device_id, led_count=reported)
|
||||
state.led_count = reported
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to sync LED count for {device_id}: {e}")
|
||||
|
||||
async def force_device_health_check(self, device_id: str) -> dict:
|
||||
"""Run an immediate health check for a device and return the result."""
|
||||
if device_id not in self._devices:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
await self._check_device_health(device_id)
|
||||
return self.get_device_health_dict(device_id)
|
||||
@@ -0,0 +1,189 @@
|
||||
"""Device test mode and idle client management mixin for ProcessorManager.
|
||||
|
||||
Extracted from processor_manager.py to keep files under 800 lines.
|
||||
"""
|
||||
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from wled_controller.core.capture.calibration import CalibrationConfig
|
||||
from wled_controller.core.devices.led_client import create_led_client
|
||||
from wled_controller.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class DeviceTestModeMixin:
|
||||
"""Mixin providing calibration test mode and idle LED client management.
|
||||
|
||||
Requires the host class to have:
|
||||
_devices: Dict[str, DeviceState]
|
||||
_processors: Dict[str, TargetProcessor]
|
||||
_idle_clients: Dict[str, object]
|
||||
"""
|
||||
|
||||
# ===== CALIBRATION TEST MODE (on device, driven by CSS calibration) =====
|
||||
|
||||
async def set_test_mode(
|
||||
self,
|
||||
device_id: str,
|
||||
edges: Dict[str, List[int]],
|
||||
calibration: Optional[CalibrationConfig] = None,
|
||||
) -> None:
|
||||
"""Set or clear calibration test mode for a device.
|
||||
|
||||
When setting test mode, pass the calibration from the CSS being tested.
|
||||
When clearing (edges={}), calibration is not needed.
|
||||
"""
|
||||
if device_id not in self._devices:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
|
||||
ds = self._devices[device_id]
|
||||
|
||||
if edges:
|
||||
ds.test_mode_active = True
|
||||
ds.test_mode_edges = {
|
||||
edge: tuple(color) for edge, color in edges.items()
|
||||
}
|
||||
if calibration is not None:
|
||||
ds.test_calibration = calibration
|
||||
await self._send_test_pixels(device_id)
|
||||
else:
|
||||
ds.test_mode_active = False
|
||||
ds.test_mode_edges = {}
|
||||
ds.test_calibration = None
|
||||
await self._send_clear_pixels(device_id)
|
||||
|
||||
async def _get_idle_client(self, device_id: str):
|
||||
"""Get or create a cached idle LED client for a device.
|
||||
|
||||
Reuses an existing connected client to avoid repeated serial
|
||||
reconnection (which triggers Arduino bootloader reset on Adalight).
|
||||
"""
|
||||
# Prefer a running processor's client (already connected)
|
||||
active = self._find_active_led_client(device_id)
|
||||
if active:
|
||||
return active
|
||||
|
||||
# Reuse cached idle client if still connected
|
||||
cached = self._idle_clients.get(device_id)
|
||||
if cached and cached.is_connected:
|
||||
return cached
|
||||
|
||||
# Create and cache a new client
|
||||
ds = self._devices[device_id]
|
||||
client = create_led_client(
|
||||
ds.device_type, ds.device_url,
|
||||
use_ddp=True, led_count=ds.led_count, baud_rate=ds.baud_rate,
|
||||
)
|
||||
await client.connect()
|
||||
self._idle_clients[device_id] = client
|
||||
return client
|
||||
|
||||
async def _close_idle_client(self, device_id: str) -> None:
|
||||
"""Close and remove the cached idle client for a device."""
|
||||
client = self._idle_clients.pop(device_id, None)
|
||||
if client:
|
||||
try:
|
||||
await client.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing idle client for {device_id}: {e}")
|
||||
|
||||
async def _send_test_pixels(self, device_id: str) -> None:
|
||||
"""Build and send test pixel array for active test edges."""
|
||||
ds = self._devices[device_id]
|
||||
|
||||
# Require calibration to know which LEDs map to which edges
|
||||
if ds.test_calibration is None:
|
||||
logger.debug(f"No calibration for test mode on {device_id}, skipping LED test")
|
||||
return
|
||||
|
||||
pixels = [(0, 0, 0)] * ds.led_count
|
||||
|
||||
for edge_name, color in ds.test_mode_edges.items():
|
||||
for seg in ds.test_calibration.segments:
|
||||
if seg.edge == edge_name:
|
||||
for i in range(seg.led_start, seg.led_start + seg.led_count):
|
||||
if i < ds.led_count:
|
||||
pixels[i] = color
|
||||
break
|
||||
|
||||
# Apply offset rotation (same as Phase 2 in PixelMapper.map_border_to_leds)
|
||||
total_leds = ds.test_calibration.get_total_leds()
|
||||
offset = ds.test_calibration.offset % total_leds if total_leds > 0 else 0
|
||||
if offset > 0:
|
||||
pixels = pixels[-offset:] + pixels[:-offset]
|
||||
|
||||
await self._send_pixels_to_device(device_id, pixels)
|
||||
|
||||
async def _send_clear_pixels(self, device_id: str) -> None:
|
||||
"""Send all-black pixels to clear LED output."""
|
||||
ds = self._devices[device_id]
|
||||
pixels = [(0, 0, 0)] * ds.led_count
|
||||
await self._send_pixels_to_device(device_id, pixels)
|
||||
|
||||
async def _send_pixels_to_device(self, device_id: str, pixels) -> None:
|
||||
"""Send pixels to a device via cached idle client.
|
||||
|
||||
Reuses a cached connection to avoid repeated serial reconnections
|
||||
(which trigger Arduino bootloader reset on Adalight devices).
|
||||
"""
|
||||
try:
|
||||
client = await self._get_idle_client(device_id)
|
||||
await client.send_pixels(pixels)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send pixels to {device_id}: {e}")
|
||||
|
||||
def _find_active_led_client(self, device_id: str):
|
||||
"""Find an active LED client for a device (from a running processor)."""
|
||||
for proc in self._processors.values():
|
||||
if proc.device_id == device_id and proc.is_running and proc.led_client:
|
||||
return proc.led_client
|
||||
return None
|
||||
|
||||
# ===== DISPLAY LOCK INFO =====
|
||||
|
||||
def is_display_locked(self, display_index: int) -> bool:
|
||||
"""Check if a display is currently being captured by any target."""
|
||||
for proc in self._processors.values():
|
||||
if proc.is_running and proc.get_display_index() == display_index:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_display_lock_info(self, display_index: int) -> Optional[str]:
|
||||
"""Get the device ID that is currently capturing from a display."""
|
||||
for proc in self._processors.values():
|
||||
if proc.is_running and proc.get_display_index() == display_index:
|
||||
return proc.device_id
|
||||
return None
|
||||
|
||||
async def clear_device(self, device_id: str) -> None:
|
||||
"""Clear LED output on a device (send black / power off)."""
|
||||
ds = self._devices.get(device_id)
|
||||
if not ds:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
try:
|
||||
await self._send_clear_pixels(device_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear device {device_id}: {e}")
|
||||
|
||||
async def _restore_device_idle_state(self, device_id: str) -> None:
|
||||
"""Restore a device to its idle state when all targets stop.
|
||||
|
||||
- For WLED: do nothing — stop() already restored the snapshot.
|
||||
- For serial: do nothing — AdalightClient.close() already sent black frame.
|
||||
"""
|
||||
ds = self._devices.get(device_id)
|
||||
if not ds or not ds.auto_shutdown:
|
||||
return
|
||||
|
||||
if self.is_device_processing(device_id):
|
||||
return
|
||||
|
||||
if ds.device_type == "wled":
|
||||
logger.info(f"Auto-restore: WLED device {device_id} restored by snapshot")
|
||||
else:
|
||||
logger.info(f"Auto-restore: {ds.device_type} device {device_id} dark (closed by processor)")
|
||||
|
||||
async def send_clear_pixels(self, device_id: str) -> None:
|
||||
"""Send all-black pixels to a device (public wrapper)."""
|
||||
await self._send_clear_pixels(device_id)
|
||||
@@ -8,13 +8,7 @@ from typing import Dict, List, Optional, Tuple
|
||||
import httpx
|
||||
|
||||
from wled_controller.core.capture.calibration import CalibrationConfig
|
||||
from wled_controller.core.devices.led_client import (
|
||||
DeviceHealth,
|
||||
check_device_health,
|
||||
create_led_client,
|
||||
get_device_capabilities,
|
||||
get_provider,
|
||||
)
|
||||
from wled_controller.core.devices.led_client import DeviceHealth
|
||||
from wled_controller.core.audio.audio_capture import AudioCaptureManager
|
||||
from wled_controller.core.processing.live_stream_manager import LiveStreamManager
|
||||
from wled_controller.core.processing.color_strip_stream_manager import ColorStripStreamManager
|
||||
@@ -28,27 +22,39 @@ from wled_controller.core.processing.target_processor import (
|
||||
)
|
||||
from wled_controller.core.processing.wled_target_processor import WledTargetProcessor
|
||||
from wled_controller.core.processing.kc_target_processor import KCTargetProcessor
|
||||
from wled_controller.core.processing.auto_restart import (
|
||||
AutoRestartMixin,
|
||||
RestartState as _RestartState,
|
||||
RESTART_MAX_ATTEMPTS as _RESTART_MAX_ATTEMPTS,
|
||||
RESTART_WINDOW_SEC as _RESTART_WINDOW_SEC,
|
||||
)
|
||||
from wled_controller.core.processing.device_health import DeviceHealthMixin
|
||||
from wled_controller.core.processing.device_test_mode import DeviceTestModeMixin
|
||||
from wled_controller.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
DEFAULT_STATE_CHECK_INTERVAL = 30 # seconds between health checks
|
||||
|
||||
# Auto-restart constants
|
||||
_RESTART_MAX_ATTEMPTS = 5 # max restarts within the window
|
||||
_RESTART_WINDOW_SEC = 300 # 5 minutes — reset counter after stable period
|
||||
_RESTART_BACKOFF_BASE = 2.0 # initial backoff seconds
|
||||
_RESTART_BACKOFF_MAX = 30.0 # cap backoff at 30s
|
||||
|
||||
|
||||
@dataclass
|
||||
class _RestartState:
|
||||
"""Per-target auto-restart tracking."""
|
||||
attempts: int = 0
|
||||
first_crash_time: float = 0.0
|
||||
last_crash_time: float = 0.0
|
||||
restart_task: Optional[asyncio.Task] = None
|
||||
enabled: bool = True # disabled on manual stop
|
||||
class ProcessorDependencies:
|
||||
"""Bundles all store and manager references needed by ProcessorManager.
|
||||
|
||||
Keeps the constructor signature stable when new stores are added.
|
||||
"""
|
||||
|
||||
picture_source_store: object = None
|
||||
capture_template_store: object = None
|
||||
pp_template_store: object = None
|
||||
pattern_template_store: object = None
|
||||
device_store: object = None
|
||||
color_strip_store: object = None
|
||||
audio_source_store: object = None
|
||||
audio_template_store: object = None
|
||||
value_source_store: object = None
|
||||
sync_clock_manager: object = None
|
||||
cspt_store: object = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -79,51 +85,58 @@ class DeviceState:
|
||||
zone_mode: str = "combined"
|
||||
|
||||
|
||||
class ProcessorManager:
|
||||
class ProcessorManager(AutoRestartMixin, DeviceHealthMixin, DeviceTestModeMixin):
|
||||
"""Manages devices and delegates target processing to TargetProcessor instances.
|
||||
|
||||
Devices are registered for health monitoring.
|
||||
Targets are registered for processing via polymorphic TargetProcessor subclasses.
|
||||
|
||||
Health monitoring is provided by DeviceHealthMixin.
|
||||
Test mode and idle client management is provided by DeviceTestModeMixin.
|
||||
"""
|
||||
|
||||
def __init__(self, picture_source_store=None, capture_template_store=None, pp_template_store=None, pattern_template_store=None, device_store=None, color_strip_store=None, audio_source_store=None, value_source_store=None, audio_template_store=None, sync_clock_manager=None, cspt_store=None):
|
||||
"""Initialize processor manager."""
|
||||
def __init__(self, deps: ProcessorDependencies):
|
||||
"""Initialize processor manager.
|
||||
|
||||
Args:
|
||||
deps: Bundled store and manager references.
|
||||
"""
|
||||
self._devices: Dict[str, DeviceState] = {}
|
||||
self._processors: Dict[str, TargetProcessor] = {}
|
||||
self._idle_clients: Dict[str, object] = {} # device_id -> cached LEDClient
|
||||
self._health_monitoring_active = False
|
||||
self._http_client: Optional[httpx.AsyncClient] = None
|
||||
self._picture_source_store = picture_source_store
|
||||
self._capture_template_store = capture_template_store
|
||||
self._pp_template_store = pp_template_store
|
||||
self._pattern_template_store = pattern_template_store
|
||||
self._device_store = device_store
|
||||
self._color_strip_store = color_strip_store
|
||||
self._audio_source_store = audio_source_store
|
||||
self._audio_template_store = audio_template_store
|
||||
self._value_source_store = value_source_store
|
||||
self._cspt_store = cspt_store
|
||||
self._picture_source_store = deps.picture_source_store
|
||||
self._capture_template_store = deps.capture_template_store
|
||||
self._pp_template_store = deps.pp_template_store
|
||||
self._pattern_template_store = deps.pattern_template_store
|
||||
self._device_store = deps.device_store
|
||||
self._color_strip_store = deps.color_strip_store
|
||||
self._audio_source_store = deps.audio_source_store
|
||||
self._audio_template_store = deps.audio_template_store
|
||||
self._value_source_store = deps.value_source_store
|
||||
self._cspt_store = deps.cspt_store
|
||||
self._live_stream_manager = LiveStreamManager(
|
||||
picture_source_store, capture_template_store, pp_template_store
|
||||
deps.picture_source_store, deps.capture_template_store, deps.pp_template_store
|
||||
)
|
||||
self._audio_capture_manager = AudioCaptureManager()
|
||||
self._sync_clock_manager = sync_clock_manager
|
||||
self._sync_clock_manager = deps.sync_clock_manager
|
||||
self._color_strip_stream_manager = ColorStripStreamManager(
|
||||
color_strip_store=color_strip_store,
|
||||
color_strip_store=deps.color_strip_store,
|
||||
live_stream_manager=self._live_stream_manager,
|
||||
audio_capture_manager=self._audio_capture_manager,
|
||||
audio_source_store=audio_source_store,
|
||||
audio_template_store=audio_template_store,
|
||||
sync_clock_manager=sync_clock_manager,
|
||||
cspt_store=cspt_store,
|
||||
audio_source_store=deps.audio_source_store,
|
||||
audio_template_store=deps.audio_template_store,
|
||||
sync_clock_manager=deps.sync_clock_manager,
|
||||
cspt_store=deps.cspt_store,
|
||||
)
|
||||
self._value_stream_manager = ValueStreamManager(
|
||||
value_source_store=value_source_store,
|
||||
value_source_store=deps.value_source_store,
|
||||
audio_capture_manager=self._audio_capture_manager,
|
||||
audio_source_store=audio_source_store,
|
||||
audio_source_store=deps.audio_source_store,
|
||||
live_stream_manager=self._live_stream_manager,
|
||||
audio_template_store=audio_template_store,
|
||||
) if value_source_store else None
|
||||
audio_template_store=deps.audio_template_store,
|
||||
) if deps.value_source_store else None
|
||||
# Wire value stream manager into CSS stream manager for composite layer brightness
|
||||
self._color_strip_stream_manager._value_stream_manager = self._value_stream_manager
|
||||
self._overlay_manager = OverlayManager()
|
||||
@@ -167,70 +180,37 @@ class ProcessorManager:
|
||||
get_device_info=self._get_device_info,
|
||||
)
|
||||
|
||||
# Default values for device-specific fields read from persistent storage
|
||||
_DEVICE_FIELD_DEFAULTS = {
|
||||
"send_latency_ms": 0, "rgbw": False, "dmx_protocol": "artnet",
|
||||
"dmx_start_universe": 0, "dmx_start_channel": 1, "espnow_peer_mac": "",
|
||||
"espnow_channel": 1, "hue_username": "", "hue_client_key": "",
|
||||
"hue_entertainment_group_id": "", "spi_speed_hz": 800000,
|
||||
"spi_led_type": "WS2812B", "chroma_device_type": "chromalink",
|
||||
"gamesense_device_type": "keyboard",
|
||||
}
|
||||
|
||||
def _get_device_info(self, device_id: str) -> Optional[DeviceInfo]:
|
||||
"""Create a DeviceInfo snapshot from the current device state."""
|
||||
ds = self._devices.get(device_id)
|
||||
if ds is None:
|
||||
return None
|
||||
# Read device-specific fields from persistent storage
|
||||
send_latency_ms = 0
|
||||
rgbw = False
|
||||
dmx_protocol = "artnet"
|
||||
dmx_start_universe = 0
|
||||
dmx_start_channel = 1
|
||||
espnow_peer_mac = ""
|
||||
espnow_channel = 1
|
||||
hue_username = ""
|
||||
hue_client_key = ""
|
||||
hue_entertainment_group_id = ""
|
||||
spi_speed_hz = 800000
|
||||
spi_led_type = "WS2812B"
|
||||
chroma_device_type = "chromalink"
|
||||
gamesense_device_type = "keyboard"
|
||||
extras = dict(self._DEVICE_FIELD_DEFAULTS)
|
||||
if self._device_store:
|
||||
try:
|
||||
dev = self._device_store.get_device(ds.device_id)
|
||||
send_latency_ms = getattr(dev, "send_latency_ms", 0)
|
||||
rgbw = getattr(dev, "rgbw", False)
|
||||
dmx_protocol = getattr(dev, "dmx_protocol", "artnet")
|
||||
dmx_start_universe = getattr(dev, "dmx_start_universe", 0)
|
||||
dmx_start_channel = getattr(dev, "dmx_start_channel", 1)
|
||||
espnow_peer_mac = getattr(dev, "espnow_peer_mac", "")
|
||||
espnow_channel = getattr(dev, "espnow_channel", 1)
|
||||
hue_username = getattr(dev, "hue_username", "")
|
||||
hue_client_key = getattr(dev, "hue_client_key", "")
|
||||
hue_entertainment_group_id = getattr(dev, "hue_entertainment_group_id", "")
|
||||
spi_speed_hz = getattr(dev, "spi_speed_hz", 800000)
|
||||
spi_led_type = getattr(dev, "spi_led_type", "WS2812B")
|
||||
chroma_device_type = getattr(dev, "chroma_device_type", "chromalink")
|
||||
gamesense_device_type = getattr(dev, "gamesense_device_type", "keyboard")
|
||||
for key, default in self._DEVICE_FIELD_DEFAULTS.items():
|
||||
extras[key] = getattr(dev, key, default)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return DeviceInfo(
|
||||
device_id=ds.device_id,
|
||||
device_url=ds.device_url,
|
||||
led_count=ds.led_count,
|
||||
device_type=ds.device_type,
|
||||
baud_rate=ds.baud_rate,
|
||||
software_brightness=ds.software_brightness,
|
||||
test_mode_active=ds.test_mode_active,
|
||||
send_latency_ms=send_latency_ms,
|
||||
rgbw=rgbw,
|
||||
zone_mode=ds.zone_mode,
|
||||
auto_shutdown=ds.auto_shutdown,
|
||||
dmx_protocol=dmx_protocol,
|
||||
dmx_start_universe=dmx_start_universe,
|
||||
dmx_start_channel=dmx_start_channel,
|
||||
espnow_peer_mac=espnow_peer_mac,
|
||||
espnow_channel=espnow_channel,
|
||||
hue_username=hue_username,
|
||||
hue_client_key=hue_client_key,
|
||||
hue_entertainment_group_id=hue_entertainment_group_id,
|
||||
spi_speed_hz=spi_speed_hz,
|
||||
spi_led_type=spi_led_type,
|
||||
chroma_device_type=chroma_device_type,
|
||||
gamesense_device_type=gamesense_device_type,
|
||||
device_id=ds.device_id, device_url=ds.device_url,
|
||||
led_count=ds.led_count, device_type=ds.device_type,
|
||||
baud_rate=ds.baud_rate, software_brightness=ds.software_brightness,
|
||||
test_mode_active=ds.test_mode_active, zone_mode=ds.zone_mode,
|
||||
auto_shutdown=ds.auto_shutdown, **extras,
|
||||
)
|
||||
|
||||
# ===== EVENT SYSTEM (state change notifications) =====
|
||||
@@ -260,7 +240,7 @@ class ProcessorManager:
|
||||
self._http_client = httpx.AsyncClient(timeout=5)
|
||||
return self._http_client
|
||||
|
||||
# ===== DEVICE MANAGEMENT (health monitoring) =====
|
||||
# ===== DEVICE MANAGEMENT =====
|
||||
|
||||
def add_device(
|
||||
self,
|
||||
@@ -475,7 +455,7 @@ class ProcessorManager:
|
||||
async def update_target_device(self, target_id: str, device_id: str):
|
||||
"""Update the device for a target.
|
||||
|
||||
If the target is currently running, performs a stop → swap → start
|
||||
If the target is currently running, performs a stop -> swap -> start
|
||||
cycle so the new device connection is established properly.
|
||||
"""
|
||||
proc = self._get_processor(target_id)
|
||||
@@ -495,7 +475,7 @@ class ProcessorManager:
|
||||
if was_running:
|
||||
await self.start_processing(target_id)
|
||||
logger.info(
|
||||
"Hot-switch complete for target %s → device %s",
|
||||
"Hot-switch complete for target %s -> device %s",
|
||||
target_id, device_id,
|
||||
)
|
||||
|
||||
@@ -742,272 +722,6 @@ class ProcessorManager:
|
||||
if proc:
|
||||
proc.remove_led_preview_client(ws)
|
||||
|
||||
# ===== CALIBRATION TEST MODE (on device, driven by CSS calibration) =====
|
||||
|
||||
async def set_test_mode(
|
||||
self,
|
||||
device_id: str,
|
||||
edges: Dict[str, List[int]],
|
||||
calibration: Optional[CalibrationConfig] = None,
|
||||
) -> None:
|
||||
"""Set or clear calibration test mode for a device.
|
||||
|
||||
When setting test mode, pass the calibration from the CSS being tested.
|
||||
When clearing (edges={}), calibration is not needed.
|
||||
"""
|
||||
if device_id not in self._devices:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
|
||||
ds = self._devices[device_id]
|
||||
|
||||
if edges:
|
||||
ds.test_mode_active = True
|
||||
ds.test_mode_edges = {
|
||||
edge: tuple(color) for edge, color in edges.items()
|
||||
}
|
||||
if calibration is not None:
|
||||
ds.test_calibration = calibration
|
||||
await self._send_test_pixels(device_id)
|
||||
else:
|
||||
ds.test_mode_active = False
|
||||
ds.test_mode_edges = {}
|
||||
ds.test_calibration = None
|
||||
await self._send_clear_pixels(device_id)
|
||||
|
||||
async def _get_idle_client(self, device_id: str):
|
||||
"""Get or create a cached idle LED client for a device.
|
||||
|
||||
Reuses an existing connected client to avoid repeated serial
|
||||
reconnection (which triggers Arduino bootloader reset on Adalight).
|
||||
"""
|
||||
# Prefer a running processor's client (already connected)
|
||||
active = self._find_active_led_client(device_id)
|
||||
if active:
|
||||
return active
|
||||
|
||||
# Reuse cached idle client if still connected
|
||||
cached = self._idle_clients.get(device_id)
|
||||
if cached and cached.is_connected:
|
||||
return cached
|
||||
|
||||
# Create and cache a new client
|
||||
ds = self._devices[device_id]
|
||||
client = create_led_client(
|
||||
ds.device_type, ds.device_url,
|
||||
use_ddp=True, led_count=ds.led_count, baud_rate=ds.baud_rate,
|
||||
)
|
||||
await client.connect()
|
||||
self._idle_clients[device_id] = client
|
||||
return client
|
||||
|
||||
async def _close_idle_client(self, device_id: str) -> None:
|
||||
"""Close and remove the cached idle client for a device."""
|
||||
client = self._idle_clients.pop(device_id, None)
|
||||
if client:
|
||||
try:
|
||||
await client.close()
|
||||
except Exception as e:
|
||||
logger.warning(f"Error closing idle client for {device_id}: {e}")
|
||||
|
||||
async def _send_test_pixels(self, device_id: str) -> None:
|
||||
"""Build and send test pixel array for active test edges."""
|
||||
ds = self._devices[device_id]
|
||||
|
||||
# Require calibration to know which LEDs map to which edges
|
||||
if ds.test_calibration is None:
|
||||
logger.debug(f"No calibration for test mode on {device_id}, skipping LED test")
|
||||
return
|
||||
|
||||
pixels = [(0, 0, 0)] * ds.led_count
|
||||
|
||||
for edge_name, color in ds.test_mode_edges.items():
|
||||
for seg in ds.test_calibration.segments:
|
||||
if seg.edge == edge_name:
|
||||
for i in range(seg.led_start, seg.led_start + seg.led_count):
|
||||
if i < ds.led_count:
|
||||
pixels[i] = color
|
||||
break
|
||||
|
||||
# Apply offset rotation (same as Phase 2 in PixelMapper.map_border_to_leds)
|
||||
total_leds = ds.test_calibration.get_total_leds()
|
||||
offset = ds.test_calibration.offset % total_leds if total_leds > 0 else 0
|
||||
if offset > 0:
|
||||
pixels = pixels[-offset:] + pixels[:-offset]
|
||||
|
||||
await self._send_pixels_to_device(device_id, pixels)
|
||||
|
||||
async def _send_clear_pixels(self, device_id: str) -> None:
|
||||
"""Send all-black pixels to clear LED output."""
|
||||
ds = self._devices[device_id]
|
||||
pixels = [(0, 0, 0)] * ds.led_count
|
||||
await self._send_pixels_to_device(device_id, pixels)
|
||||
|
||||
async def _send_pixels_to_device(self, device_id: str, pixels) -> None:
|
||||
"""Send pixels to a device via cached idle client.
|
||||
|
||||
Reuses a cached connection to avoid repeated serial reconnections
|
||||
(which trigger Arduino bootloader reset on Adalight devices).
|
||||
"""
|
||||
try:
|
||||
client = await self._get_idle_client(device_id)
|
||||
await client.send_pixels(pixels)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to send pixels to {device_id}: {e}")
|
||||
|
||||
def _find_active_led_client(self, device_id: str):
|
||||
"""Find an active LED client for a device (from a running processor)."""
|
||||
for proc in self._processors.values():
|
||||
if proc.device_id == device_id and proc.is_running and proc.led_client:
|
||||
return proc.led_client
|
||||
return None
|
||||
|
||||
# ===== DISPLAY LOCK INFO =====
|
||||
|
||||
def is_display_locked(self, display_index: int) -> bool:
|
||||
"""Check if a display is currently being captured by any target."""
|
||||
for proc in self._processors.values():
|
||||
if proc.is_running and proc.get_display_index() == display_index:
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_display_lock_info(self, display_index: int) -> Optional[str]:
|
||||
"""Get the device ID that is currently capturing from a display."""
|
||||
for proc in self._processors.values():
|
||||
if proc.is_running and proc.get_display_index() == display_index:
|
||||
return proc.device_id
|
||||
return None
|
||||
|
||||
async def clear_device(self, device_id: str) -> None:
|
||||
"""Clear LED output on a device (send black / power off)."""
|
||||
ds = self._devices.get(device_id)
|
||||
if not ds:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
try:
|
||||
await self._send_clear_pixels(device_id)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to clear device {device_id}: {e}")
|
||||
|
||||
async def _restore_device_idle_state(self, device_id: str) -> None:
|
||||
"""Restore a device to its idle state when all targets stop.
|
||||
|
||||
- For WLED: do nothing — stop() already restored the snapshot.
|
||||
- For serial: do nothing — AdalightClient.close() already sent black frame.
|
||||
"""
|
||||
ds = self._devices.get(device_id)
|
||||
if not ds or not ds.auto_shutdown:
|
||||
return
|
||||
|
||||
if self.is_device_processing(device_id):
|
||||
return
|
||||
|
||||
if ds.device_type == "wled":
|
||||
logger.info(f"Auto-restore: WLED device {device_id} restored by snapshot")
|
||||
else:
|
||||
logger.info(f"Auto-restore: {ds.device_type} device {device_id} dark (closed by processor)")
|
||||
|
||||
# ===== AUTO-RESTART =====
|
||||
|
||||
def _on_task_done(self, target_id: str, task: asyncio.Task) -> None:
|
||||
"""Task done callback — detects crashes and schedules auto-restart."""
|
||||
# Ignore graceful cancellation (manual stop)
|
||||
if task.cancelled():
|
||||
return
|
||||
|
||||
exc = task.exception()
|
||||
if exc is None:
|
||||
return # Clean exit (shouldn't happen, but harmless)
|
||||
|
||||
rs = self._restart_states.get(target_id)
|
||||
if not rs or not rs.enabled:
|
||||
return # Auto-restart disabled (manual stop was called)
|
||||
|
||||
now = time.monotonic()
|
||||
|
||||
# Reset counter if previous crash window expired
|
||||
if rs.first_crash_time and (now - rs.first_crash_time) > _RESTART_WINDOW_SEC:
|
||||
rs.attempts = 0
|
||||
rs.first_crash_time = 0.0
|
||||
|
||||
rs.attempts += 1
|
||||
rs.last_crash_time = now
|
||||
if not rs.first_crash_time:
|
||||
rs.first_crash_time = now
|
||||
|
||||
if rs.attempts > _RESTART_MAX_ATTEMPTS:
|
||||
logger.error(
|
||||
f"[AUTO-RESTART] Target {target_id} crashed {rs.attempts} times "
|
||||
f"in {now - rs.first_crash_time:.0f}s — giving up"
|
||||
)
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_exhausted": True,
|
||||
})
|
||||
return
|
||||
|
||||
backoff = min(
|
||||
_RESTART_BACKOFF_BASE * (2 ** (rs.attempts - 1)),
|
||||
_RESTART_BACKOFF_MAX,
|
||||
)
|
||||
logger.warning(
|
||||
f"[AUTO-RESTART] Target {target_id} crashed (attempt {rs.attempts}/"
|
||||
f"{_RESTART_MAX_ATTEMPTS}), restarting in {backoff:.1f}s"
|
||||
)
|
||||
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_in": backoff,
|
||||
"auto_restart_attempt": rs.attempts,
|
||||
})
|
||||
|
||||
# Schedule the restart (runs in the event loop)
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
logger.error(f"[AUTO-RESTART] No running event loop for {target_id}")
|
||||
return
|
||||
|
||||
rs.restart_task = loop.create_task(self._auto_restart(target_id, backoff))
|
||||
|
||||
async def _auto_restart(self, target_id: str, delay: float) -> None:
|
||||
"""Wait for backoff delay, then restart the target processor."""
|
||||
try:
|
||||
await asyncio.sleep(delay)
|
||||
except asyncio.CancelledError:
|
||||
logger.info(f"[AUTO-RESTART] Restart cancelled for {target_id}")
|
||||
return
|
||||
|
||||
rs = self._restart_states.get(target_id)
|
||||
if not rs or not rs.enabled:
|
||||
logger.info(f"[AUTO-RESTART] Restart aborted for {target_id} (disabled)")
|
||||
return
|
||||
|
||||
proc = self._processors.get(target_id)
|
||||
if proc is None:
|
||||
logger.warning(f"[AUTO-RESTART] Target {target_id} no longer registered")
|
||||
return
|
||||
if proc.is_running:
|
||||
logger.info(f"[AUTO-RESTART] Target {target_id} already running, skipping")
|
||||
return
|
||||
|
||||
logger.info(f"[AUTO-RESTART] Restarting target {target_id} (attempt {rs.attempts})")
|
||||
try:
|
||||
await self.start_processing(target_id)
|
||||
except Exception as e:
|
||||
logger.error(f"[AUTO-RESTART] Failed to restart {target_id}: {e}")
|
||||
self.fire_event({
|
||||
"type": "state_change",
|
||||
"target_id": target_id,
|
||||
"processing": False,
|
||||
"crashed": True,
|
||||
"auto_restart_error": str(e),
|
||||
})
|
||||
|
||||
# ===== LIFECYCLE =====
|
||||
|
||||
async def stop_all(self):
|
||||
@@ -1055,120 +769,6 @@ class ProcessorManager:
|
||||
|
||||
logger.info("Stopped all processors")
|
||||
|
||||
# ===== HEALTH MONITORING =====
|
||||
|
||||
async def start_health_monitoring(self):
|
||||
"""Start background health checks for all registered devices."""
|
||||
self._health_monitoring_active = True
|
||||
for device_id in self._devices:
|
||||
self._start_device_health_check(device_id)
|
||||
await self._metrics_history.start()
|
||||
logger.info("Started health monitoring for all devices")
|
||||
|
||||
async def stop_health_monitoring(self):
|
||||
"""Stop all background health checks."""
|
||||
self._health_monitoring_active = False
|
||||
for device_id in list(self._devices.keys()):
|
||||
self._stop_device_health_check(device_id)
|
||||
logger.info("Stopped health monitoring for all devices")
|
||||
|
||||
def _start_device_health_check(self, device_id: str):
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
# Skip periodic health checks for virtual devices (always online)
|
||||
if "health_check" not in get_device_capabilities(state.device_type):
|
||||
from datetime import datetime, timezone
|
||||
state.health = DeviceHealth(online=True, latency_ms=0.0, last_checked=datetime.now(timezone.utc))
|
||||
return
|
||||
if state.health_task and not state.health_task.done():
|
||||
return
|
||||
state.health_task = asyncio.create_task(self._health_check_loop(device_id))
|
||||
|
||||
def _stop_device_health_check(self, device_id: str):
|
||||
state = self._devices.get(device_id)
|
||||
if not state or not state.health_task:
|
||||
return
|
||||
state.health_task.cancel()
|
||||
state.health_task = None
|
||||
|
||||
def _device_is_processing(self, device_id: str) -> bool:
|
||||
"""Check if any target is actively streaming to this device."""
|
||||
return any(
|
||||
p.device_id == device_id and p.is_running
|
||||
for p in self._processors.values()
|
||||
)
|
||||
|
||||
def _is_device_streaming(self, device_id: str) -> bool:
|
||||
"""Check if any running processor targets this device."""
|
||||
for proc in self._processors.values():
|
||||
if getattr(proc, 'device_id', None) == device_id and proc.is_running:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def _health_check_loop(self, device_id: str):
|
||||
"""Background loop that periodically checks a device.
|
||||
|
||||
Uses adaptive intervals: 10s for actively streaming devices,
|
||||
60s for idle devices, to balance responsiveness with overhead.
|
||||
"""
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
|
||||
ACTIVE_INTERVAL = 10 # streaming devices — faster detection
|
||||
IDLE_INTERVAL = 60 # idle devices — less overhead
|
||||
|
||||
try:
|
||||
while self._health_monitoring_active:
|
||||
await self._check_device_health(device_id)
|
||||
interval = ACTIVE_INTERVAL if self._is_device_streaming(device_id) else IDLE_INTERVAL
|
||||
await asyncio.sleep(interval)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as e:
|
||||
logger.error(f"Fatal error in health check loop for {device_id}: {e}")
|
||||
|
||||
async def _check_device_health(self, device_id: str):
|
||||
"""Check device health. Also auto-syncs LED count if changed."""
|
||||
state = self._devices.get(device_id)
|
||||
if not state:
|
||||
return
|
||||
prev_online = state.health.online
|
||||
client = await self._get_http_client()
|
||||
state.health = await check_device_health(
|
||||
state.device_type, state.device_url, client, state.health,
|
||||
)
|
||||
|
||||
# Fire event when online status changes
|
||||
if state.health.online != prev_online:
|
||||
self.fire_event({
|
||||
"type": "device_health_changed",
|
||||
"device_id": device_id,
|
||||
"online": state.health.online,
|
||||
"latency_ms": state.health.latency_ms,
|
||||
})
|
||||
|
||||
# Auto-sync LED count
|
||||
reported = state.health.device_led_count
|
||||
if reported and reported != state.led_count and self._device_store:
|
||||
old_count = state.led_count
|
||||
logger.info(
|
||||
f"Device {device_id} LED count changed: {old_count} → {reported}"
|
||||
)
|
||||
try:
|
||||
self._device_store.update_device(device_id, led_count=reported)
|
||||
state.led_count = reported
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to sync LED count for {device_id}: {e}")
|
||||
|
||||
async def force_device_health_check(self, device_id: str) -> dict:
|
||||
"""Run an immediate health check for a device and return the result."""
|
||||
if device_id not in self._devices:
|
||||
raise ValueError(f"Device {device_id} not found")
|
||||
await self._check_device_health(device_id)
|
||||
return self.get_device_health_dict(device_id)
|
||||
|
||||
# ===== HELPERS =====
|
||||
|
||||
def has_device(self, device_id: str) -> bool:
|
||||
@@ -1179,10 +779,6 @@ class ProcessorManager:
|
||||
"""Get device state, returning None if not registered."""
|
||||
return self._devices.get(device_id)
|
||||
|
||||
async def send_clear_pixels(self, device_id: str) -> None:
|
||||
"""Send all-black pixels to a device (public wrapper)."""
|
||||
await self._send_clear_pixels(device_id)
|
||||
|
||||
def get_processor(self, target_id: str) -> Optional[TargetProcessor]:
|
||||
"""Look up a processor by target_id, returning None if not found."""
|
||||
return self._processors.get(target_id)
|
||||
|
||||
Reference in New Issue
Block a user