Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 alexei.dolgolyov
|
02e2ea37f3 | ||
|
alexei.dolgolyov
|
fdc9201660 | ||
|
alexei.dolgolyov
|
5686ae5468 | ||
|
alexei.dolgolyov
|
9960f15a1b | ||
|
alexei.dolgolyov
|
397a53ed1c | ||
|
alexei.dolgolyov
|
1c1bbe2551 | ||
|
alexei.dolgolyov
|
68040173c6 |
@@ -1,308 +0,0 @@
|
||||
# Plan: Android on-device audio capture
|
||||
|
||||
> Status: proposed plan (not yet approved). No code changes. Last updated 2026-06-01.
|
||||
|
||||
## Context
|
||||
|
||||
LedGrab's audio-reactive features (music analyzer, audio value sources, band filters)
|
||||
depend on capturing an audio stream and running it through `AudioAnalyzer`
|
||||
(`server/src/ledgrab/core/audio/analysis.py`). On desktop this is fed by **WASAPI**
|
||||
(Windows) or **Sounddevice/PortAudio** (cross-platform). On the **experimental
|
||||
Android-TV build** neither is available — `sounddevice` has no Chaquopy wheel and PortAudio
|
||||
isn't bundled — so `core/audio/__init__.py` registers only `DemoAudioEngine`, and
|
||||
audio-reactive lighting is effectively dead on Android.
|
||||
|
||||
Android does not need PortAudio: the platform exposes **`AudioPlaybackCapture`** (API 29+),
|
||||
which captures system playback audio and **takes a `MediaProjection` token — the very token
|
||||
the app already obtains for screen capture** (`ScreenCapture(projection, …)`). This plan adds
|
||||
a push-based Android audio engine so the TV box can drive sound-reactive lighting from its own
|
||||
media playback, at parity with how desktop audio feeds the analyzer.
|
||||
|
||||
The design mirrors the working screen-capture bridge
|
||||
(`mediaprojection_engine.py` ↔ `ScreenCapture.kt` ↔ `PythonBridge`) and the existing audio
|
||||
engine abstraction (`AudioCaptureEngine` / `AudioCaptureStreamBase` /
|
||||
`AudioEngineRegistry`). **No new Python dependencies** (`numpy` is already bundled) → no
|
||||
Chaquopy / `build.gradle.kts` `pip {}` changes.
|
||||
|
||||
---
|
||||
|
||||
## Approach
|
||||
|
||||
A new **push-based** audio engine registered in the existing `AudioEngineRegistry`:
|
||||
|
||||
- **Python:** `AndroidAudioEngine` + `AndroidAudioCaptureStream` mirroring `SounddeviceEngine`,
|
||||
but `read_chunk()` pops PCM from a module-level queue that **Kotlin fills** (mirror of
|
||||
`mediaprojection_engine.push_frame`). High `ENGINE_PRIORITY` so
|
||||
`AudioEngineRegistry.get_best_available_engine()` selects it on Android. The existing
|
||||
`ManagedAudioStream` capture loop and `AudioAnalyzer` consume `read_chunk()` unchanged.
|
||||
- **Android:** an `AudioCapture` helper using `AudioRecord` + `AudioPlaybackCaptureConfiguration`
|
||||
(reusing `CaptureService`'s `MediaProjection`), pushing float32 PCM to Python. Mic
|
||||
(`AudioSource.MIC`) fallback. Wired into `CaptureService` next to `ScreenCapture`.
|
||||
|
||||
```
|
||||
[media playback] → AudioRecord (AudioPlaybackCapture, reuses MediaProjection)
|
||||
→ AudioCapture.kt → PythonBridge.pushAudio(pcmFloat32, frames, channels)
|
||||
→ android_audio_engine.push_samples() [module-level queue]
|
||||
→ AndroidAudioCaptureStream.read_chunk() → ManagedAudioStream → AudioAnalyzer [unchanged]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Part A — Python (server)
|
||||
|
||||
**New file: `server/src/ledgrab/core/audio/android_audio_engine.py`** — mirror
|
||||
`mediaprojection_engine.py` (queue + configure + push) and `sounddevice_engine.py` (engine/stream shape):
|
||||
|
||||
```python
|
||||
import queue
|
||||
import numpy as np
|
||||
from typing import Any, Dict, List
|
||||
from ledgrab.core.audio.base import AudioCaptureEngine, AudioCaptureStreamBase, AudioDeviceInfo
|
||||
from ledgrab.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
_pcm_queue: "queue.Queue[np.ndarray]" = queue.Queue(maxsize=8)
|
||||
_sample_rate = 48000
|
||||
_channels = 2
|
||||
_chunk_size = 1024
|
||||
_active = False
|
||||
|
||||
def configure(sample_rate: int, channels: int, chunk_size: int) -> None:
|
||||
"""Called from Kotlin before audio frames start flowing. Drains stale PCM."""
|
||||
global _sample_rate, _channels, _chunk_size, _active
|
||||
while not _pcm_queue.empty():
|
||||
try: _pcm_queue.get_nowait()
|
||||
except queue.Empty: break
|
||||
_sample_rate, _channels, _chunk_size = sample_rate, channels, chunk_size
|
||||
_active = True
|
||||
|
||||
def push_samples(pcm_float32: bytes) -> None:
|
||||
"""Push one interleaved float32 PCM chunk from Kotlin. Drops oldest if full."""
|
||||
samples = np.frombuffer(pcm_float32, dtype=np.float32)
|
||||
try:
|
||||
_pcm_queue.put_nowait(samples)
|
||||
except queue.Full:
|
||||
try: _pcm_queue.get_nowait()
|
||||
except queue.Empty: pass
|
||||
try: _pcm_queue.put_nowait(samples)
|
||||
except queue.Full: pass
|
||||
|
||||
def shutdown() -> None:
|
||||
global _active
|
||||
_active = False
|
||||
|
||||
|
||||
class AndroidAudioCaptureStream(AudioCaptureStreamBase):
|
||||
@property
|
||||
def channels(self) -> int: return _channels
|
||||
@property
|
||||
def sample_rate(self) -> int: return _sample_rate
|
||||
@property
|
||||
def chunk_size(self) -> int: return _chunk_size
|
||||
def initialize(self) -> None:
|
||||
if not _active:
|
||||
raise RuntimeError("Android audio engine not configured (only valid in-app).")
|
||||
self._initialized = True
|
||||
def cleanup(self) -> None:
|
||||
self._initialized = False
|
||||
def read_chunk(self) -> np.ndarray | None:
|
||||
try:
|
||||
return _pcm_queue.get(timeout=0.1) # 1-D float32 interleaved
|
||||
except queue.Empty:
|
||||
return None
|
||||
|
||||
|
||||
class AndroidAudioEngine(AudioCaptureEngine):
|
||||
ENGINE_TYPE = "android_playback"
|
||||
ENGINE_PRIORITY = 100 # highest on Android (demo is lower)
|
||||
@classmethod
|
||||
def is_available(cls) -> bool:
|
||||
from ledgrab.utils.platform import is_android
|
||||
return is_android() and _active
|
||||
@classmethod
|
||||
def get_default_config(cls) -> Dict[str, Any]:
|
||||
return {"sample_rate": _sample_rate, "channels": _channels, "chunk_size": _chunk_size}
|
||||
@classmethod
|
||||
def enumerate_devices(cls) -> List[AudioDeviceInfo]:
|
||||
if not cls.is_available(): return []
|
||||
return [AudioDeviceInfo(index=0, name="Android playback (system audio)",
|
||||
is_input=True, is_loopback=True,
|
||||
channels=_channels, default_samplerate=float(_sample_rate))]
|
||||
@classmethod
|
||||
def create_stream(cls, device_index, is_loopback, config) -> AndroidAudioCaptureStream:
|
||||
return AndroidAudioCaptureStream(device_index, is_loopback, {**cls.get_default_config(), **config})
|
||||
```
|
||||
|
||||
**Modify `server/src/ledgrab/core/audio/__init__.py`** — register behind a guarded import,
|
||||
matching the existing `_has_wasapi` / `_has_sounddevice` pattern:
|
||||
|
||||
```python
|
||||
try:
|
||||
from ledgrab.core.audio.android_audio_engine import AndroidAudioEngine
|
||||
_has_android_audio = True
|
||||
except ImportError:
|
||||
_has_android_audio = False
|
||||
...
|
||||
if _has_android_audio:
|
||||
AudioEngineRegistry.register(AndroidAudioEngine)
|
||||
```
|
||||
|
||||
**Reused, unchanged:** `AudioEngineRegistry.get_best_available_engine()` (picks by priority),
|
||||
`ManagedAudioStream._capture_loop()` (`audio_capture.py`), `AudioAnalyzer`, the audio value
|
||||
sources, and the device-enumeration endpoints. The Android engine appears as one loopback
|
||||
device named "Android playback (system audio)".
|
||||
|
||||
---
|
||||
|
||||
## Part B — Android (Kotlin + manifest)
|
||||
|
||||
**New file: `android/app/src/main/java/com/ledgrab/android/AudioCapture.kt`**
|
||||
|
||||
Mirrors `ScreenCapture.kt`, taking the same `MediaProjection`:
|
||||
|
||||
```kotlin
|
||||
class AudioCapture(
|
||||
private val projection: MediaProjection,
|
||||
private val bridge: PythonBridge,
|
||||
private val sampleRate: Int = 48000,
|
||||
private val channels: Int = 2,
|
||||
private val chunkFrames: Int = 1024,
|
||||
)
|
||||
```
|
||||
|
||||
- `start()` (API 29+, MediaProjection mode):
|
||||
- Build `AudioPlaybackCaptureConfiguration(projection)` adding usages
|
||||
`USAGE_MEDIA`, `USAGE_GAME`, `USAGE_UNKNOWN` (the capturable set).
|
||||
- `AudioRecord.Builder().setAudioPlaybackCaptureConfig(cfg)` with
|
||||
`AudioFormat(ENCODING_PCM_FLOAT, sampleRate, CHANNEL_IN_STEREO)`.
|
||||
- On a dedicated `HandlerThread`, loop `audioRecord.read(floatBuf, …, READ_BLOCKING)` →
|
||||
wrap into a little-endian float32 `ByteArray` (reusable buffer, like `ScreenCapture`'s
|
||||
`frameBuffer`) → `bridge.pushAudio(bytes, framesRead, channels)`.
|
||||
- `stop()`: stop/release `AudioRecord`, quit the thread.
|
||||
- **Mic fallback** (`startMic()`): `AudioSource.MIC` for root mode (no MediaProjection) or
|
||||
API < 29. Used only when playback capture is unavailable.
|
||||
|
||||
**Modify `android/app/src/main/java/com/ledgrab/android/PythonBridge.kt`** — add the audio
|
||||
push path (same shape as `pushFrame`, with a cached PyObject handle):
|
||||
|
||||
```kotlin
|
||||
@Volatile private var androidAudioEngine: PyObject? = null
|
||||
|
||||
fun configureAudio(sampleRate: Int, channels: Int, chunkFrames: Int) {
|
||||
val engine = Python.getInstance().getModule("ledgrab.core.audio.android_audio_engine")
|
||||
engine.callAttr("configure", sampleRate, channels, chunkFrames)
|
||||
androidAudioEngine = engine
|
||||
}
|
||||
fun pushAudio(pcmFloat32: ByteArray, frames: Int, channels: Int) {
|
||||
if (!running) return
|
||||
androidAudioEngine?.let {
|
||||
try { it.callAttr("push_samples", pcmFloat32) }
|
||||
catch (e: Exception) { Log.w(TAG, "pushAudio failed: ${e.message}") }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Modify `android/app/src/main/java/com/ledgrab/android/CaptureService.kt`** — in the
|
||||
MediaProjection start path (where `ScreenCapture` is created with the projection), if
|
||||
`RECORD_AUDIO` is granted and API ≥ 29, also `bridge.configureAudio(...)` and start an
|
||||
`AudioCapture(projection, bridge)`. Stop/release it in `onDestroy` alongside `ScreenCapture`.
|
||||
Root path → optional mic fallback (or skip; see Risks).
|
||||
|
||||
**Modify `android/app/src/main/AndroidManifest.xml`:**
|
||||
```xml
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<!-- For mic-mode foreground capture on API 34+ (playback capture is covered by the
|
||||
existing mediaProjection FGS type): -->
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
|
||||
```
|
||||
The existing `CaptureService` already declares `foregroundServiceType="mediaProjection|specialUse"`
|
||||
and holds `FOREGROUND_SERVICE_MEDIA_PROJECTION`; add `microphone` to the type only if mic
|
||||
fallback is implemented.
|
||||
|
||||
**Modify `MainActivity.kt`** — request `RECORD_AUDIO` at runtime alongside the existing
|
||||
`ensureNotificationPermission()` (POST_NOTIFICATIONS) flow, before starting capture. Capture
|
||||
proceeds without audio if denied (graceful degradation).
|
||||
|
||||
---
|
||||
|
||||
## Orchestration decision (the main trade-off)
|
||||
|
||||
Desktop starts audio capture **on demand** when an audio-reactive source is acquired
|
||||
(`AudioCaptureManager.acquire`). On Android, PCM only flows if Kotlin has set up `AudioRecord`.
|
||||
|
||||
- **MVP (recommended):** start `AudioCapture` when `CaptureService` starts (if `RECORD_AUDIO`
|
||||
granted + MediaProjection mode + API ≥ 29) and push continuously; the bounded queue drops
|
||||
frames when no audio source consumes them. Simplest; modest extra CPU.
|
||||
- **Future optimization:** on-demand start/stop signaled Python→Kotlin (Chaquopy can call
|
||||
Kotlin, as `BleBridge`/`UsbSerialBridge` show) so `AudioRecord` runs only while an
|
||||
audio-reactive source is active. Defer unless CPU/battery on low-end boxes warrants it.
|
||||
|
||||
---
|
||||
|
||||
## What does NOT change
|
||||
|
||||
- **Frontend / API** — audio engine + device selection, the music analyzer UI, and audio value
|
||||
sources are engine-agnostic; the Android engine shows up via the existing device enumeration.
|
||||
- **`build.gradle.kts` / Chaquopy pip block** — no new Python packages.
|
||||
- **Audio analysis pipeline** — `AudioAnalyzer`, band filters, `ManagedAudioStream` untouched.
|
||||
|
||||
---
|
||||
|
||||
## Files
|
||||
|
||||
**Create**
|
||||
- `server/src/ledgrab/core/audio/android_audio_engine.py`
|
||||
- `android/app/src/main/java/com/ledgrab/android/AudioCapture.kt`
|
||||
- `server/tests/core/audio/test_android_audio_engine.py`
|
||||
|
||||
**Modify**
|
||||
- `server/src/ledgrab/core/audio/__init__.py` — guarded import + registry registration.
|
||||
- `android/app/src/main/java/com/ledgrab/android/PythonBridge.kt` — `configureAudio` + `pushAudio`.
|
||||
- `android/app/src/main/java/com/ledgrab/android/CaptureService.kt` — start/stop `AudioCapture`.
|
||||
- `android/app/src/main/java/com/ledgrab/android/MainActivity.kt` — request `RECORD_AUDIO`.
|
||||
- `android/app/src/main/AndroidManifest.xml` — `RECORD_AUDIO` (+ mic FGS if mic fallback).
|
||||
|
||||
---
|
||||
|
||||
## Tests (Python — run on desktop CI, no Android device needed)
|
||||
|
||||
New `server/tests/core/audio/test_android_audio_engine.py`:
|
||||
|
||||
- `configure()` then `push_samples()` → `read_chunk()` returns the same float32 samples;
|
||||
queue drops oldest when full (push > maxsize).
|
||||
- `AndroidAudioEngine.is_available()` is `False` until `configure()` and only on Android
|
||||
(monkeypatch `ledgrab.utils.platform.is_android`); `True` after.
|
||||
- `enumerate_devices()` returns exactly one loopback device when active, `[]` otherwise.
|
||||
- Integration: with `is_android()` patched true + `configure()`, `get_best_available_engine()`
|
||||
returns `"android_playback"` (priority beats demo), and a stream created via
|
||||
`AudioEngineRegistry.create_stream("android_playback", 0, True, {})` yields pushed chunks.
|
||||
- Registry isolation: use `AudioEngineRegistry.clear_registry()` / re-register in fixtures so
|
||||
desktop engines aren't disturbed.
|
||||
|
||||
## Verification
|
||||
|
||||
1. **Python:** `py -3.13 -m pytest tests/core/audio/test_android_audio_engine.py --no-cov -q`
|
||||
(from `server/`), then the full suite.
|
||||
2. **Lint:** `ruff check src/ tests/ --fix` (from `server/`).
|
||||
3. **Android build:** `./gradlew :app:assembleDebug` (from `android/`).
|
||||
4. **On device/emulator (manual):** install APK → grant `RECORD_AUDIO` + screen-capture consent
|
||||
→ start capture → play non-DRM media (e.g. a local video / YouTube web) → create an
|
||||
audio-reactive value source bound to a strip → confirm the LEDs react to the audio, and the
|
||||
Android playback device appears in audio device enumeration.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- **DRM opt-out:** Netflix/Disney+/etc. set audio as non-capturable; `AudioPlaybackCapture`
|
||||
yields silence for them. Works for non-DRM media and the device's own audio. Document in UI.
|
||||
- **API 29 minimum** for playback capture (minSdk is 24). API 24–28 and root mode (no
|
||||
MediaProjection) → mic fallback only, or audio unsupported. Gate cleanly + log.
|
||||
- **`RECORD_AUDIO`** is a runtime "dangerous" permission — must be requested; capture must
|
||||
degrade gracefully when denied.
|
||||
- **Format:** request `ENCODING_PCM_FLOAT` so Kotlin pushes float32 matching
|
||||
`read_chunk()`'s contract (1-D interleaved float32, length = frames × channels). If a device
|
||||
rejects float, capture 16-bit PCM and convert (`/32768.0`) before pushing.
|
||||
- **Latency/CPU:** small `chunkFrames` (e.g. 1024 @ 48 kHz ≈ 21 ms) keeps reactivity tight;
|
||||
continuous capture (MVP) adds modest CPU on low-end boxes — see the orchestration trade-off.
|
||||
- **R8/ProGuard:** minify is disabled and the Python module is resolved by string from Kotlin;
|
||||
no new keep-rules needed.
|
||||
@@ -1,181 +0,0 @@
|
||||
# Android (TV) — Missing Functionality Assessment
|
||||
|
||||
> Status: review/feasibility document. No code changes. Last updated 2026-06-01.
|
||||
|
||||
## Context
|
||||
|
||||
LedGrab ships an **experimental on-device Android-TV build**: a Kotlin shell that
|
||||
embeds the Python FastAPI server via **Chaquopy**, with Kotlin↔Python **bridges**
|
||||
(`PythonBridge`, `BleBridge`, `UsbSerialBridge`). Several desktop features are
|
||||
unavailable on this build because their Python backends rely on native libraries
|
||||
that have no Android/Chaquopy wheels (`mss`, `dxcam`, `sounddevice`/PortAudio,
|
||||
`opencv`, `nvidia-ml-py`, `winrt`, `dbus-next`), or on OS facilities Android
|
||||
sandboxes differently.
|
||||
|
||||
The README "Feature support by OS" table now carries an Android column reflecting
|
||||
this. This document assesses **whether each missing feature can be added**, how, and
|
||||
whether it's worth it.
|
||||
|
||||
### The enabling pattern (why most of this is feasible)
|
||||
|
||||
Every desktop capability that's "missing" on Android is missing only because of a
|
||||
*native dependency*, not because the capability is impossible. Android exposes the
|
||||
same capability through a platform API, and the codebase already has the bridge
|
||||
shape to plug it in:
|
||||
|
||||
> **Bridge pattern:** a Kotlin component captures an event/buffer → pushes it across
|
||||
> the Chaquopy JNI boundary into a **module-level receiver** in a small Python engine
|
||||
> → an existing engine/stream consumes it unchanged.
|
||||
|
||||
Reference implementation: `server/src/ledgrab/core/capture_engines/mediaprojection_engine.py`
|
||||
(`configure()` + `push_frame()` + a bounded `queue.Queue`) ↔
|
||||
`android/app/src/main/java/com/ledgrab/android/ScreenCapture.kt` ↔
|
||||
`PythonBridge.pushFrame()`. Screen capture already works on Android this exact way.
|
||||
|
||||
So for most missing features the work is: **add a Kotlin capture source + a thin
|
||||
Python receiver engine mirroring that pattern.**
|
||||
|
||||
---
|
||||
|
||||
## Current Android capability matrix
|
||||
|
||||
| Feature | Desktop | Android (TV) today | Missing? |
|
||||
| ------- | ------- | ------------------ | -------- |
|
||||
| Screen capture | DXCam/WGC/MSS | ✅ MediaProjection + root `screenrecord` | No |
|
||||
| LED transports (network/USB-serial/BLE) | ✅ | ✅ (USB via Android driver, BLE via Android bridge) | No |
|
||||
| System metrics | psutil | ✅ CPU/RAM/battery/thermal via `/proc`, `/sys` (`AndroidMetricsProvider`) | No |
|
||||
| **Audio capture** | WASAPI / Sounddevice | ❌ no PortAudio | **Yes** |
|
||||
| Notification capture | WinRT / D-Bus | ✅ NotificationListenerService → `push_notification()` | No (implemented) |
|
||||
| Webcam capture | OpenCV | ✅ Camera2 + on-demand bridge (`AndroidCameraEngine`) | No (implemented) |
|
||||
| GPU monitoring | NVML | ❌ no NVIDIA GPU | Marginal |
|
||||
| Capture from *another* Android phone | scrcpy/ADB | ❌ | Skip (redundant) |
|
||||
| Automation: window/process conditions | Windows ctypes | ❌ sandboxed | Partial |
|
||||
| Monitor names / multi-display | WMI / generic | Single built-in display | Low value |
|
||||
|
||||
---
|
||||
|
||||
## Per-feature feasibility
|
||||
|
||||
### 🔊 Audio capture — **FEASIBLE, HIGH VALUE** ⭐ (detailed plan exists)
|
||||
|
||||
- **Blocker:** only `sounddevice`/PortAudio is missing — not the capability.
|
||||
- **Android path:** `AudioPlaybackCapture` (API 29+) captures system playback audio and
|
||||
**takes a `MediaProjection` token — which the app already obtains for screen capture.**
|
||||
Kotlin `AudioRecord` → push PCM (float32) → a new push-based `AndroidAudioEngine`
|
||||
mirroring `mediaprojection_engine.py`, registered in `core/audio/__init__.py`, feeding
|
||||
the existing `AudioAnalyzer` unchanged. Mic (`AudioSource.MIC`) is the fallback.
|
||||
- **Effort:** moderate. **Value:** high — music/sound-reactive lighting is a flagship use
|
||||
on a TV box. **No new Python deps.**
|
||||
- ⚠️ DRM-protected apps (Netflix etc.) opt out of playback capture; works for non-DRM
|
||||
media and the device's own audio. Root mode (no MediaProjection) → mic-only.
|
||||
- 📄 **See `android-audio-capture-plan.md`** for the full implementation plan.
|
||||
|
||||
### 🔔 Notification capture — **IMPLEMENTED** ✅ (shipped)
|
||||
|
||||
- **Android is the *best* platform for this:** `NotificationListenerService` is the native,
|
||||
event-push mechanism (no polling).
|
||||
- **Path:** a `NotificationListenerService` resolves the posting app's display label and
|
||||
pushes it via a module-level `push_notification()` into the existing
|
||||
`os_notification_listener.py` pipeline (a new push-based `_AndroidBackend` alongside
|
||||
`_WindowsBackend`/`_LinuxBackend`). Existing `NotificationColorStripSource` filters,
|
||||
per-app colors/sounds, and the history endpoint all work unchanged. **No new Python deps.**
|
||||
- **Permission:** user enables "Notification access" in Settings (`ACTION_NOTIFICATION_LISTENER_SETTINGS`);
|
||||
no runtime-permission popup.
|
||||
- **Effort:** moderate. **Value:** high.
|
||||
- ✅ **Implemented** on branch `feature/android-notification-capture`: a push-based
|
||||
`_AndroidBackend` + module-level `push_notification()` in `os_notification_listener.py`,
|
||||
a Kotlin `LedGrabNotificationListener` (NLS), and prompt-once permission UX. App-name
|
||||
parity — only the resolved app label crosses the JNI boundary, never the notification
|
||||
title/body. ⚠️ App labels can differ across OSes (Windows `display_name` / Linux D-Bus
|
||||
`app_name` / Android `getApplicationLabel`), so desktop-configured per-app colors/filters
|
||||
may need re-matching on Android.
|
||||
|
||||
### 📷 Webcam capture — **IMPLEMENTED** ✅ (shipped)
|
||||
|
||||
- **Blocker** was `opencv-python-headless` (no Chaquopy cp311 wheel) — but capture doesn't
|
||||
*need* OpenCV. Implemented with **Camera2** + `ImageReader` in Kotlin pushing RGB frames
|
||||
through the same bridge as MediaProjection into a new `AndroidCameraEngine`.
|
||||
- **Path:** a Kotlin `CameraBridge` singleton (Camera2) enumerates cameras and **opens the
|
||||
camera on demand** (only while a capture source is active — driven Python→Kotlin via the
|
||||
`BleBridge`/`UsbSerialBridge` pattern), converts each frame YUV_420_888→RGB, and pushes it
|
||||
into a push-based `AndroidCameraEngine` (`core/capture_engines/android_camera_engine.py`)
|
||||
that mirrors `mediaprojection_engine.py`. Cameras surface as selectable "displays" exactly
|
||||
like the desktop OpenCV `CameraEngine`; the data-driven capture-template UI (engine list +
|
||||
`resolution` config + display picker) needs **no changes**. **No new Python deps; no new
|
||||
Gradle deps** (Camera2 is in-platform).
|
||||
- **Permission:** `CAMERA` requested at capture-start, gated on `FEATURE_CAMERA_ANY` so
|
||||
camera-less TV boxes never see the prompt; graceful degradation when denied. The service is
|
||||
promoted with the `camera` FGS type (+ `FOREGROUND_SERVICE_CAMERA`) **only when CAMERA is
|
||||
already granted**, so backgrounded capture keeps working without risking a failed service
|
||||
start on camera-less boxes. (Unlike audio playback capture, the camera can't ride the
|
||||
MediaProjection token, so it needs its own FGS type to survive backgrounding.)
|
||||
- **Effort:** moderate. **Value:** low (TVs rarely have cameras), but the implementation reuses
|
||||
existing infrastructure end-to-end. **Priority `0`** so it's never auto-selected over
|
||||
MediaProjection — chosen explicitly via `engine_type="android_camera"`.
|
||||
- ⚠️ **MVP scope / limitations:** webcam capture works **while LedGrab capture is running**
|
||||
(no camera-only server path on Android); one camera active at a time; `"auto"` picks a
|
||||
balanced output size (not the sensor max) to keep per-frame YUV→RGB cheap; USB-UVC webcams
|
||||
appear only if the device routes them through Camera2 (varies by box); no frame-rotation
|
||||
correction.
|
||||
- 📄 **See `android-webcam-capture-plan.md`** for the full implementation notes.
|
||||
|
||||
### 🎮 GPU monitoring — **MARGINAL, SKIP FOR NOW**
|
||||
|
||||
- NVML is desktop-NVIDIA only. Android GPU load lives in **vendor-specific sysfs**
|
||||
(Adreno `/sys/class/kgsl/kgsl-3d0/gpubusy`, Mali `/sys/class/devfreq/*.mali/...`),
|
||||
inconsistent and often root-only.
|
||||
- CPU/RAM/battery/thermal are **already** covered by `AndroidMetricsProvider`. A best-effort
|
||||
GPU-load reader could be added to that provider, but reliability is poor and value is low.
|
||||
|
||||
### 🪟 Automation: window/process conditions — **PARTIAL**
|
||||
|
||||
- Android forbids full window/process enumeration (`getRunningTasks` restricted since API 21+).
|
||||
- **Obtainable:** the *current foreground app package* via `UsageStatsManager` (needs the
|
||||
`PACKAGE_USAGE_STATS` special access) or an `AccessibilityService`.
|
||||
- So "when <app> is in the foreground → scene X" is feasible (mirrors
|
||||
`automations/platform_detector.py`, which currently returns empty off-Windows); full
|
||||
window-title matching is **not**. **Effort:** moderate. **Value:** moderate (per-app scenes
|
||||
on a TV box).
|
||||
|
||||
### 📱 Capture from *another* Android phone (scrcpy/ADB) — **SKIP**
|
||||
|
||||
- Impractical and redundant: no `adb` binary in Chaquopy, TV boxes can't reliably host an
|
||||
adb server, and the device already captures its **own** screen via MediaProjection.
|
||||
|
||||
### 🖥️ Monitor names / multi-display — **LOW VALUE**
|
||||
|
||||
- `DisplayManager` can report a better display name and enumerate secondary (HDMI) displays,
|
||||
but MediaProjection captures the default display; capturing a secondary display is more
|
||||
involved and rarely useful on a single-screen box.
|
||||
|
||||
---
|
||||
|
||||
## Prioritization
|
||||
|
||||
| Priority | Feature | Effort | Value | New Python deps | Status |
|
||||
| -------- | ------- | ------ | ----- | --------------- | ------ |
|
||||
| 1 | Notification capture | Moderate | High | None | **✅ Implemented** |
|
||||
| 2 | Audio capture | Moderate | High | None | **✅ Implemented** |
|
||||
| 4 | Webcam capture (Camera2) | Moderate | Low | None | **✅ Implemented** |
|
||||
| 3 | Automation: foreground-app condition | Moderate | Moderate | None | Idea (only remaining) |
|
||||
| — | GPU load (vendor sysfs) | Low–Med | Low | None | Not recommended |
|
||||
| — | Capture from another phone | — | — | — | Won't do |
|
||||
| — | Multi-display / monitor names | Low | Low | None | Not recommended |
|
||||
|
||||
**Status:** notifications, audio, **and webcam** are all shipped — each reuses existing
|
||||
infrastructure (bridge pattern, the MediaProjection consent token / process-global
|
||||
`Python.getInstance()`, the capture/audio/notification pipelines) and adds **zero** Python
|
||||
dependencies, so none risks the Chaquopy `--no-deps` build constraint documented in
|
||||
`CLAUDE.md`. The only remaining idea is the **foreground-app automation condition** (moderate
|
||||
value); GPU load, another-phone capture, and multi-display remain not-recommended / won't-do.
|
||||
|
||||
## Cross-cutting notes
|
||||
|
||||
- **No `build.gradle.kts` / Chaquopy pip impact** for notifications or audio — both use Android
|
||||
platform APIs (Kotlin) + stdlib/`numpy` (already bundled) on the Python side.
|
||||
- **Per-instance `PythonBridge`:** `PythonBridge` is created per `CaptureService` instance, so
|
||||
system-bound services (e.g. a `NotificationListenerService`) call Python via the
|
||||
process-global `Python.getInstance()` rather than borrowing that bridge.
|
||||
- **Permissions are the recurring friction**, not the capture: audio needs `RECORD_AUDIO` +
|
||||
(for playback capture) a MediaProjection token; notifications need the "Notification access"
|
||||
settings toggle; foreground-app automation needs `PACKAGE_USAGE_STATS`.
|
||||
@@ -1,168 +0,0 @@
|
||||
# Plan: Android on-device webcam capture
|
||||
|
||||
> Status: **implemented** on branch `feature/android-webcam-capture`. Last updated 2026-06-02.
|
||||
|
||||
## Context
|
||||
|
||||
LedGrab captures webcams on desktop through OpenCV (`cv2.VideoCapture`) in
|
||||
`server/src/ledgrab/core/capture_engines/camera_engine.py`. On the **experimental Android-TV
|
||||
build**, `opencv-python-headless` has no Chaquopy cp311 wheel, so the camera engine never
|
||||
loads and cameras are unusable on-device.
|
||||
|
||||
Android doesn't need OpenCV to capture a camera: the platform exposes **Camera2**
|
||||
(`android.hardware.camera2`), and the codebase already has the bridge shape to plug a Kotlin
|
||||
capture source into a push-based Python engine. This feature adds an on-device camera engine
|
||||
so a USB/integrated camera can drive ambient lighting, at parity with how the desktop OpenCV
|
||||
camera engine feeds the pipeline.
|
||||
|
||||
The design mirrors the working screen-capture bridge
|
||||
(`mediaprojection_engine.py` ↔ `ScreenCapture.kt`) and the just-shipped audio engine
|
||||
(`android_audio_engine.py` ↔ `AudioCapture.kt`). **No new Python dependencies** (numpy already
|
||||
bundled) and **no new Gradle dependencies** (Camera2 is in-platform) → no Chaquopy /
|
||||
`build.gradle.kts` changes.
|
||||
|
||||
## Approach
|
||||
|
||||
A new **push-based** capture engine registered in the existing `EngineRegistry`, plus a Kotlin
|
||||
`CameraBridge` that opens the camera **on demand**:
|
||||
|
||||
```
|
||||
[capture source acquired] → AndroidCameraCaptureStream.initialize()
|
||||
→ android_camera_engine.start_camera(index, w, h) [guarded jclass]
|
||||
→ CameraBridge.startCamera(index, w, h) [Camera2 open + session]
|
||||
→ onImageAvailable → YUV_420_888→RGB (stride-aware) → push_frame(rgbBytes, w, h)
|
||||
→ android_camera_engine [module-level queue] → AndroidCameraCaptureStream.capture_frame()
|
||||
→ ScreenCaptureLiveStream → processing pipeline [unchanged]
|
||||
|
||||
[capture source released] → AndroidCameraCaptureStream.cleanup()
|
||||
→ android_camera_engine.stop_camera() → CameraBridge.stopCamera() [releases the camera]
|
||||
```
|
||||
|
||||
The camera is **only open while a camera source is active** — the camera-in-use indicator and
|
||||
battery cost are bounded to actual use, unlike always-on screen/audio capture. This on-demand
|
||||
control reuses the synchronous Python→Kotlin singleton pattern of `BleBridge`/`UsbSerialBridge`.
|
||||
|
||||
## Selection path (why nothing downstream changes)
|
||||
|
||||
Webcams on desktop are a `ScreenCapturePictureSource` (`stream_type="raw"`) bound to a capture
|
||||
template whose `engine_type="camera"` + a `display_index`. `live_stream_manager`
|
||||
`_create_screen_capture_live_stream` reads `engine_type` from the template and calls
|
||||
`EngineRegistry.create_stream(engine_type, display_index, config)`. Android adds
|
||||
`engine_type="android_camera"` — the **same path**. The frontend
|
||||
(`static/js/features/streams-capture-templates.ts`) is fully data-driven: the engine list,
|
||||
the `resolution` config dropdown (keyed by field name), and the camera picker
|
||||
(`/config/displays?engine_type=android_camera`, since `HAS_OWN_DISPLAYS=True`) all work with
|
||||
no frontend changes.
|
||||
|
||||
## Part A — Python (`core/capture_engines/android_camera_engine.py`)
|
||||
|
||||
Mirrors `mediaprojection_engine.py` (module-level `queue.Queue` + `push_frame` + `_last_frame`
|
||||
fallback + drop-oldest) and the desktop `CameraEngine` shape (cameras as displays,
|
||||
`resolution` config).
|
||||
|
||||
- `_camera_bridge()` — lazy, `is_android()`-guarded `from java import jclass;
|
||||
jclass("com.ledgrab.android.CameraBridge").INSTANCE`. **Never imported at module load** (this
|
||||
module imports on desktop CI). Mirrors `core/devices/android_ble_transport.py`.
|
||||
- `list_cameras()` → parses `CameraBridge.listCameras()` JSON into
|
||||
`[{"index","name","facing"}]`; `_enumerate_cameras()` caches it (30 s TTL).
|
||||
- `push_frame(rgb_bytes, w, h)` → `np.frombuffer(...uint8)` reshape **`(h, w, 3)`** (RGB, 3
|
||||
B/px — NOT the RGBA `(h,w,4)` of the screen engine) → `.copy()` → drop-oldest enqueue. A
|
||||
short/malformed buffer is dropped, never reshape-crashes.
|
||||
- `start_camera(index, w, h) -> bool` / `stop_camera(index)` → guarded bridge calls.
|
||||
- `AndroidCameraEngine`: `ENGINE_TYPE="android_camera"`, `ENGINE_PRIORITY=0` (never
|
||||
auto-selected over MediaProjection=100 — explicit `engine_type` only), `HAS_OWN_DISPLAYS=True`,
|
||||
`is_available()=is_android() and ≥1 enumerated camera`, `get_config_choices()` exposes
|
||||
`resolution` (same presets as desktop).
|
||||
- `AndroidCameraCaptureStream`: `initialize()` parses `resolution` → `start_camera(...)` (raises
|
||||
if it returns False), drains stale frames; `capture_frame()` pops queue / returns `_last_frame`;
|
||||
`cleanup()` → `stop_camera(...)`.
|
||||
|
||||
Registered in `capture_engines/__init__.py` behind a guarded import (mirrors the
|
||||
mediaprojection block).
|
||||
|
||||
## Part B — Android (`CameraBridge.kt`)
|
||||
|
||||
`object CameraBridge` (mirrors `BleBridge`):
|
||||
|
||||
- `init(context)` — from `LedGrabApp.onCreate` (context only, no camera opened).
|
||||
- `listCameras(): String` — JSON array from `CameraManager.cameraIdList` + `LENS_FACING`
|
||||
(front/back/external). No CAMERA permission needed.
|
||||
- `startCamera(index, width, height): Boolean` — checks CAMERA permission; resolves cameraId;
|
||||
picks the supported YUV size closest to the request (balanced default ≤1280×720 for "auto");
|
||||
opens device + capture session on a private `HandlerThread`, blocking until configured
|
||||
(`runBlocking { withTimeout { ... } }` over `suspendCancellableCoroutine`-wrapped Camera2
|
||||
callbacks); sets a repeating preview request. Returns false (no throw across JNI) on
|
||||
permission/range/configure failure. Closes any prior camera first.
|
||||
- `onImageAvailable` → paced (≈20 fps) → stride-aware **YUV_420_888→RGB** (BT.601 fixed-point,
|
||||
reused plane + RGB buffers) → push to the cached `android_camera_engine` module handle.
|
||||
- `stopCamera()` — stops repeating, closes session/device/reader, idempotent.
|
||||
|
||||
## Part C — Wiring + permission + manifest
|
||||
|
||||
- `LedGrabApp.kt` — `CameraBridge.init(this)` next to `BleBridge.init`.
|
||||
- `MainActivity.kt` — `ensureCameraPermission()` (mirror `ensureAudioPermission`): request
|
||||
`CAMERA` iff `hasSystemFeature(FEATURE_CAMERA_ANY)`; called from both `startCaptureService`
|
||||
(MediaProjection path) and `startRootCaptureService` (root path). Fire-and-forget.
|
||||
- `AndroidManifest.xml` — `<uses-permission CAMERA>` + `<uses-feature camera.any required=false>`
|
||||
+ `<uses-permission FOREGROUND_SERVICE_CAMERA>`, and `camera` added to the `CaptureService`
|
||||
`foregroundServiceType` union (`mediaProjection|specialUse|camera`).
|
||||
- `CaptureService.onStartCommand` — on API 34+, OR `FOREGROUND_SERVICE_TYPE_CAMERA` into the
|
||||
promotion type **only when CAMERA is already granted**. Unlike audio playback capture (which
|
||||
rides the MediaProjection token under the mediaProjection type), the camera has no such
|
||||
coupling, so without its own FGS type Android 14+ revokes camera access once the app is
|
||||
backgrounded. The conditional guard avoids a failed `startForeground` (which would kill the
|
||||
whole service) on a camera-less / not-yet-granted box. If CAMERA is granted later, the camera
|
||||
type takes effect on the next Start.
|
||||
- No `proguard-rules.pro` change — the blanket `-keep class com.ledgrab.android.** { *; }`
|
||||
already covers `CameraBridge`, and R8/minify is disabled.
|
||||
|
||||
## What does NOT change
|
||||
|
||||
- **Frontend / API** — data-driven engine list, config, and display picker.
|
||||
- **`build.gradle.kts` / Chaquopy pip block** — no new Python or Gradle packages.
|
||||
- **Processing pipeline** — `ScreenCaptureLiveStream`, filters, color-strip sources unchanged.
|
||||
|
||||
## Files
|
||||
|
||||
**Create**
|
||||
- `server/src/ledgrab/core/capture_engines/android_camera_engine.py`
|
||||
- `android/app/src/main/java/com/ledgrab/android/CameraBridge.kt`
|
||||
- `server/tests/core/test_android_camera_engine.py`
|
||||
|
||||
**Modify**
|
||||
- `server/src/ledgrab/core/capture_engines/__init__.py` — guarded import + registration.
|
||||
- `android/app/src/main/java/com/ledgrab/android/LedGrabApp.kt` — `CameraBridge.init`.
|
||||
- `android/app/src/main/java/com/ledgrab/android/MainActivity.kt` — `ensureCameraPermission`.
|
||||
- `android/app/src/main/AndroidManifest.xml` — `CAMERA` + `camera.any`.
|
||||
|
||||
## Tests (Python — desktop CI, no device)
|
||||
|
||||
`server/tests/core/test_android_camera_engine.py`: push→capture round-trips RGB `(h,w,3)`;
|
||||
drop-oldest when full; `_last_frame` fallback on empty; short-buffer never crashes;
|
||||
`initialize()` opens with parsed/auto resolution and raises on open-failure / off-Android;
|
||||
`cleanup()` closes once (idempotent); `is_available()` gating (android + cameras); display
|
||||
enumeration; priority 0 never beats MediaProjection; create-via-registry yields a pushed frame.
|
||||
|
||||
## Verification
|
||||
|
||||
1. **Python:** `py -3.13 -m pytest tests/core/test_android_camera_engine.py --no-cov -q`, then
|
||||
the full suite (1880 passed, 2 skipped; 15 new).
|
||||
2. **Lint:** `ruff check src/ tests/ --fix` — clean.
|
||||
3. **Android build:** `./gradlew :app:assembleDebug` — BUILD SUCCESSFUL.
|
||||
4. **On device (manual):** install APK → Start capture → grant CAMERA → create a capture
|
||||
template with engine `android_camera` + a camera display + a ScreenCapture source bound to
|
||||
a strip → confirm LEDs react to the camera feed and the camera indicator only lights while
|
||||
the source is active.
|
||||
|
||||
## Risks / notes
|
||||
|
||||
- **MVP scope:** webcam works **while LedGrab capture is running** (the Python server only runs
|
||||
inside `CaptureService`; there is no camera-only start path on Android).
|
||||
- **One camera at a time:** `startCamera` closes any previously-open camera first.
|
||||
- **`"auto"` resolution** picks a balanced output size (~720p), not the sensor max, to keep the
|
||||
per-frame YUV→RGB conversion cheap on low-end TV boxes.
|
||||
- **USB-UVC webcams** appear only if the device exposes them through Camera2 (`LENS_FACING_EXTERNAL`),
|
||||
which varies by box; an explicit UVC library would be a separate, larger effort.
|
||||
- **No frame-rotation correction** — sensor orientation is not applied (ambient color sampling
|
||||
is largely orientation-tolerant); could be added later.
|
||||
- **CAMERA denied** → the engine reports no usable camera and capture proceeds without it.
|
||||
@@ -115,7 +115,7 @@ LedGrab runs as a desktop / server application:
|
||||
| Notification capture | WinRT | dbus (Linux) | NotificationListenerService |
|
||||
| Monitor names | Friendly names (WMI) | Generic ("Display 0") | Single built-in display |
|
||||
| LED transports | Network, USB-serial, BLE | Network, USB-serial, BLE | Network, USB-serial (Android driver), BLE (Android bridge) |
|
||||
| Automation: window/process conditions | Supported | Partial | — |
|
||||
| Automation: window/process conditions | Supported | Partial | Foreground-app condition (UsageStatsManager) |
|
||||
|
||||
## Requirements
|
||||
|
||||
|
||||
@@ -65,6 +65,18 @@
|
||||
service start. -->
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
|
||||
<!-- PACKAGE_USAGE_STATS — read the foreground app for the "Application"
|
||||
automation rule (foreground app -> activate scene) via UsageStatsManager.
|
||||
A special-access permission: it can't be granted at runtime; the user
|
||||
toggles it under Settings > Usage access (opened from MainActivity).
|
||||
tools:ignore="ProtectedPermissions" silences the build warning that this
|
||||
is a system/signature-level permission — it is honoured as a user-grantable
|
||||
special access. NO QUERY_ALL_PACKAGES is needed: matching only compares the
|
||||
foreground package NAME, and the app picker uses LauncherApps. -->
|
||||
<uses-permission
|
||||
android:name="android.permission.PACKAGE_USAGE_STATS"
|
||||
tools:ignore="ProtectedPermissions" />
|
||||
|
||||
<!-- Autostart on boot — BootReceiver spawns CaptureService in root
|
||||
mode so capture resumes without the user touching the remote. -->
|
||||
<uses-permission android:name="android.permission.RECEIVE_BOOT_COMPLETED" />
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package com.ledgrab.android
|
||||
|
||||
import android.app.AppOpsManager
|
||||
import android.app.usage.UsageEvents
|
||||
import android.app.usage.UsageStatsManager
|
||||
import android.content.Context
|
||||
import android.content.pm.LauncherApps
|
||||
import android.os.Build
|
||||
import android.os.Process
|
||||
import android.util.Log
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONObject
|
||||
|
||||
/**
|
||||
* Foreground-app + installed-app bridge exposed to the Python server via Chaquopy.
|
||||
*
|
||||
* Backs the Android implementation of the "Application" automation rule
|
||||
* (foreground app -> activate scene). Desktop detects the foreground process via
|
||||
* Win32 ctypes in ``platform_detector.py``; Android has no such API, so this
|
||||
* bridge wraps two in-platform services into synchronous calls a Python thread
|
||||
* can invoke (Chaquopy proxy threads are real OS threads):
|
||||
*
|
||||
* - [getForegroundPackage] via [UsageStatsManager] (needs PACKAGE_USAGE_STATS,
|
||||
* a special-access permission granted from Settings — see MainActivity).
|
||||
* - [listLaunchableApps] via [LauncherApps] for the automation editor's app
|
||||
* picker (no QUERY_ALL_PACKAGES needed — getActivityList is the sanctioned
|
||||
* launchable-app enumeration API).
|
||||
* - [hasUsageAccess] so the server / UI can detect the missing grant.
|
||||
*
|
||||
* Detection only ever string-compares the foreground *package name*, so no label
|
||||
* resolution / package visibility is required at match time.
|
||||
*
|
||||
* Python callers access the singleton via
|
||||
* `jclass("com.ledgrab.android.ForegroundAppBridge").INSTANCE` — see
|
||||
* `server/src/ledgrab/core/automations/platform_detector.py`.
|
||||
*/
|
||||
object ForegroundAppBridge {
|
||||
private const val TAG = "ForegroundAppBridge"
|
||||
|
||||
// Trailing window for queryEvents. queryEvents reports discrete foreground
|
||||
// transitions (not "current app"), and events can lag a few seconds, so we
|
||||
// look back far enough to reliably catch the latest MOVE_TO_FOREGROUND while
|
||||
// staying recent enough not to report a stale app on the ~1s automation tick.
|
||||
private const val WINDOW_MS = 10_000L
|
||||
|
||||
@Volatile private var appContext: Context? = null
|
||||
|
||||
/** Called once from [LedGrabApp.onCreate] to bind the application context. */
|
||||
@JvmStatic
|
||||
fun init(context: Context) {
|
||||
appContext = context.applicationContext
|
||||
}
|
||||
|
||||
/**
|
||||
* Package name of the most recently foregrounded app, or null when none is
|
||||
* found in the trailing window, Usage Access is not granted, or on any error.
|
||||
* Never throws across the JNI boundary.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getForegroundPackage(): String? {
|
||||
val ctx = appContext ?: run {
|
||||
Log.w(TAG, "getForegroundPackage: context not bound (init not called)")
|
||||
return null
|
||||
}
|
||||
return try {
|
||||
val usm = ctx.getSystemService(Context.USAGE_STATS_SERVICE) as? UsageStatsManager
|
||||
?: return null
|
||||
val end = System.currentTimeMillis()
|
||||
val events = usm.queryEvents(end - WINDOW_MS, end)
|
||||
val event = UsageEvents.Event()
|
||||
var latestPkg: String? = null
|
||||
var latestTs = Long.MIN_VALUE
|
||||
while (events.hasNextEvent()) {
|
||||
events.getNextEvent(event)
|
||||
// ACTIVITY_RESUMED (API 29+) shares the value of the legacy
|
||||
// MOVE_TO_FOREGROUND constant, so the single check covers both.
|
||||
// >= (not >) so that on an exact-timestamp tie the later-iterated
|
||||
// event wins — events arrive chronologically, so that is the most
|
||||
// recent foreground transition.
|
||||
if (event.eventType == UsageEvents.Event.MOVE_TO_FOREGROUND &&
|
||||
event.timeStamp >= latestTs
|
||||
) {
|
||||
latestTs = event.timeStamp
|
||||
latestPkg = event.packageName
|
||||
}
|
||||
}
|
||||
latestPkg
|
||||
} catch (e: Exception) {
|
||||
// SecurityException when access is missing, plus any service error.
|
||||
Log.w(TAG, "getForegroundPackage failed: ${e.message}")
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the user has granted Usage Access (PACKAGE_USAGE_STATS) to this app. */
|
||||
@JvmStatic
|
||||
fun hasUsageAccess(): Boolean {
|
||||
val ctx = appContext ?: return false
|
||||
return try {
|
||||
val appOps = ctx.getSystemService(Context.APP_OPS_SERVICE) as? AppOpsManager
|
||||
?: return false
|
||||
val mode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
appOps.unsafeCheckOpNoThrow(
|
||||
AppOpsManager.OPSTR_GET_USAGE_STATS, Process.myUid(), ctx.packageName,
|
||||
)
|
||||
} else {
|
||||
@Suppress("DEPRECATION")
|
||||
appOps.checkOpNoThrow(
|
||||
AppOpsManager.OPSTR_GET_USAGE_STATS, Process.myUid(), ctx.packageName,
|
||||
)
|
||||
}
|
||||
mode == AppOpsManager.MODE_ALLOWED
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "hasUsageAccess failed: ${e.message}")
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Launchable apps as a JSON array string the Python server parses:
|
||||
* `[{"package":"com.netflix.mediaclient","label":"Netflix"}, ...]`
|
||||
*
|
||||
* Uses [LauncherApps.getActivityList] (launcher + leanback launchables) —
|
||||
* no QUERY_ALL_PACKAGES. De-duplicated by package, sorted by label.
|
||||
* Returns `[]` on any error.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun listLaunchableApps(): String {
|
||||
val arr = JSONArray()
|
||||
val ctx = appContext ?: run {
|
||||
Log.w(TAG, "listLaunchableApps: context not bound (init not called)")
|
||||
return arr.toString()
|
||||
}
|
||||
try {
|
||||
val launcher = ctx.getSystemService(Context.LAUNCHER_APPS_SERVICE) as? LauncherApps
|
||||
?: return arr.toString()
|
||||
val seen = HashSet<String>()
|
||||
val items = ArrayList<Pair<String, String>>()
|
||||
for (info in launcher.getActivityList(null, Process.myUserHandle())) {
|
||||
val pkg = info.applicationInfo?.packageName ?: continue
|
||||
if (!seen.add(pkg)) continue
|
||||
val label = info.label?.toString().takeUnless { it.isNullOrBlank() } ?: pkg
|
||||
items.add(pkg to label)
|
||||
}
|
||||
items.sortBy { it.second.lowercase() }
|
||||
for ((pkg, label) in items) {
|
||||
arr.put(JSONObject().put("package", pkg).put("label", label))
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "listLaunchableApps failed: ${e.message}")
|
||||
}
|
||||
return arr.toString()
|
||||
}
|
||||
}
|
||||
@@ -54,6 +54,10 @@ class LedGrabApp : Application() {
|
||||
// Bind application context for the camera bridge so Python can
|
||||
// enumerate cameras and open them on demand (webcam capture).
|
||||
CameraBridge.init(this)
|
||||
// Bind application context for the foreground-app bridge so Python can
|
||||
// detect the foreground app (Application automation rule) and list
|
||||
// launchable apps for the editor's picker.
|
||||
ForegroundAppBridge.init(this)
|
||||
|
||||
// Pre-warm the API key on a background thread. First-launch
|
||||
// generation does a SharedPreferences.commit() (synchronous
|
||||
|
||||
@@ -69,6 +69,7 @@ class MainActivity : Activity() {
|
||||
private lateinit var autostartCheck: CheckBox
|
||||
private lateinit var autostartPrefs: AutostartPrefs
|
||||
private lateinit var grantNotificationButton: Button
|
||||
private lateinit var grantUsageAccessButton: Button
|
||||
|
||||
// Running-state views (lazy-inflated via ViewStub).
|
||||
private lateinit var runningPanelStub: ViewStub
|
||||
@@ -113,6 +114,7 @@ class MainActivity : Activity() {
|
||||
versionText = findViewById(R.id.version_text)
|
||||
autostartCheck = findViewById(R.id.autostart_check)
|
||||
grantNotificationButton = findViewById(R.id.grant_notification_button)
|
||||
grantUsageAccessButton = findViewById(R.id.grant_usage_access_button)
|
||||
|
||||
val versionName = packageManager.getPackageInfo(packageName, 0).versionName
|
||||
versionText.text = getString(R.string.version_prefix, versionName ?: "?")
|
||||
@@ -134,9 +136,10 @@ class MainActivity : Activity() {
|
||||
}
|
||||
|
||||
grantNotificationButton.setOnClickListener { openNotificationListenerSettings() }
|
||||
grantUsageAccessButton.setOnClickListener { openUsageAccessSettings() }
|
||||
toggleButton.setOnClickListener { startCapture() }
|
||||
|
||||
updateNotificationAccessUi()
|
||||
updateStoppedPermissionButtons()
|
||||
updateUI()
|
||||
}
|
||||
|
||||
@@ -166,7 +169,7 @@ class MainActivity : Activity() {
|
||||
if (CaptureService.isRunning) {
|
||||
updateUI()
|
||||
} else {
|
||||
updateNotificationAccessUi()
|
||||
updateStoppedPermissionButtons()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -544,6 +547,26 @@ class MainActivity : Activity() {
|
||||
}.onFailure { Log.w(TAG, "Notification-access settings unavailable: ${it.message}") }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether Usage Access (PACKAGE_USAGE_STATS) is granted — needed by the
|
||||
* foreground-app automation rule. Delegates to the bridge's AppOps check.
|
||||
*/
|
||||
private fun isUsageAccessGranted(): Boolean = ForegroundAppBridge.hasUsageAccess()
|
||||
|
||||
/**
|
||||
* Open the system Usage-Access screen so the user can grant LedGrab access
|
||||
* for the foreground-app automation rule. Falls back to the generic Settings
|
||||
* screen on TV-box OEM builds that strip the dedicated intent.
|
||||
*/
|
||||
private fun openUsageAccessSettings() {
|
||||
runCatching {
|
||||
startActivity(Intent(Settings.ACTION_USAGE_ACCESS_SETTINGS))
|
||||
}.onFailure {
|
||||
Log.w(TAG, "Usage-access settings unavailable: ${it.message}")
|
||||
runCatching { startActivity(Intent(Settings.ACTION_SETTINGS)) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prompt-once-then-remember: the first time capture starts without
|
||||
* notification-listener access, open the settings screen so the user can
|
||||
@@ -559,20 +582,24 @@ class MainActivity : Activity() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Show the "Grant notification access" button only while access is missing,
|
||||
* then re-wire the D-pad focus chain. Called on create and on resume
|
||||
* (access can change in Settings while we're backgrounded).
|
||||
* Show each "Grant <permission> access" button only while that access is
|
||||
* missing, then re-wire the D-pad focus chain. Called on create and on resume
|
||||
* (access can change in Settings while we're backgrounded). The usage-access
|
||||
* button is a passive affordance (no auto-prompt at capture start) — the
|
||||
* primary guidance is the web-UI banner when an Android app rule needs it.
|
||||
*/
|
||||
private fun updateNotificationAccessUi() {
|
||||
private fun updateStoppedPermissionButtons() {
|
||||
if (!::grantNotificationButton.isInitialized) return
|
||||
grantNotificationButton.visibility =
|
||||
if (isNotificationAccessGranted()) View.GONE else View.VISIBLE
|
||||
grantUsageAccessButton.visibility =
|
||||
if (isUsageAccessGranted()) View.GONE else View.VISIBLE
|
||||
wireStoppedFocusChain()
|
||||
}
|
||||
|
||||
/**
|
||||
* Link the visible stopped-panel controls into a single up/down D-pad chain.
|
||||
* Both optional controls (the grant-access button and the root-only autostart
|
||||
* The optional controls (the grant-access buttons and the root-only autostart
|
||||
* checkbox) may be GONE, so the chain is computed from whatever is visible —
|
||||
* a static nextFocus pointing at a GONE view would strand the focus on a TV
|
||||
* remote.
|
||||
@@ -581,6 +608,7 @@ class MainActivity : Activity() {
|
||||
val chain = listOfNotNull(
|
||||
toggleButton,
|
||||
grantNotificationButton.takeIf { it.visibility == View.VISIBLE },
|
||||
grantUsageAccessButton.takeIf { it.visibility == View.VISIBLE },
|
||||
autostartCheck.takeIf { it.visibility == View.VISIBLE },
|
||||
)
|
||||
chain.forEachIndexed { i, view ->
|
||||
|
||||
@@ -81,6 +81,21 @@
|
||||
android:focusableInTouchMode="true"
|
||||
android:visibility="gone" />
|
||||
|
||||
<!-- Shown only while Usage Access is missing (needed by the foreground-app
|
||||
automation rule). Like the grant-notification button, its D-pad focus
|
||||
chain is wired at runtime (wireStoppedFocusChain). -->
|
||||
<Button
|
||||
android:id="@+id/grant_usage_access_button"
|
||||
style="@style/Widget.LedGrab.Button.Secondary"
|
||||
android:layout_width="320dp"
|
||||
android:layout_height="56dp"
|
||||
android:layout_marginTop="20dp"
|
||||
android:text="@string/btn_grant_usage_access"
|
||||
android:textSize="18sp"
|
||||
android:focusable="true"
|
||||
android:focusableInTouchMode="true"
|
||||
android:visibility="gone" />
|
||||
|
||||
<CheckBox
|
||||
android:id="@+id/autostart_check"
|
||||
android:layout_width="wrap_content"
|
||||
|
||||
@@ -27,4 +27,5 @@
|
||||
<string name="notification_text">Веб-интерфейс: %1$s</string>
|
||||
<string name="notification_listener_label">Захват уведомлений LedGrab</string>
|
||||
<string name="btn_grant_notification_access">Разрешить доступ к уведомлениям</string>
|
||||
<string name="btn_grant_usage_access">Разрешить доступ к статистике использования</string>
|
||||
</resources>
|
||||
|
||||
@@ -27,4 +27,5 @@
|
||||
<string name="notification_text">Web界面:%1$s</string>
|
||||
<string name="notification_listener_label">LedGrab 通知捕获</string>
|
||||
<string name="btn_grant_notification_access">授予通知访问权限</string>
|
||||
<string name="btn_grant_usage_access">授予使用情况访问权限</string>
|
||||
</resources>
|
||||
|
||||
@@ -27,4 +27,5 @@
|
||||
<string name="notification_text">Web UI: %1$s</string>
|
||||
<string name="notification_listener_label">LedGrab notification capture</string>
|
||||
<string name="btn_grant_notification_access">Grant notification access</string>
|
||||
<string name="btn_grant_usage_access">Grant usage access</string>
|
||||
</resources>
|
||||
|
||||
@@ -15,11 +15,11 @@ auth:
|
||||
# - LAN requests are REJECTED with 401 (security default)
|
||||
# To enable LAN access, uncomment the example below and replace the value
|
||||
# with a secret you generated yourself (e.g. `openssl rand -hex 32`).
|
||||
# The previous default `dev: "development-key-change-in-production"` has
|
||||
# been removed — it shipped as a publicly-known token and any deployment
|
||||
# that still uses it grants full LAN access to anyone on the network.
|
||||
api_keys:
|
||||
dev: "development-key-change-in-production"
|
||||
# Do NOT ship a hard-coded key here — a publicly-known token grants full
|
||||
# LAN access to anyone on the network.
|
||||
api_keys: {}
|
||||
# api_keys:
|
||||
# my-client: "replace-with-output-of-openssl-rand-hex-32"
|
||||
|
||||
# Storage paths default to ./data relative to the server's working directory.
|
||||
# Set LEDGRAB_DATA_DIR in the environment to point at a different data root
|
||||
|
||||
@@ -39,8 +39,11 @@ from ledgrab.api.schemas.system import (
|
||||
DisplayListResponse,
|
||||
GpuInfo,
|
||||
HealthResponse,
|
||||
InstalledAppItem,
|
||||
InstalledAppsResponse,
|
||||
PerformanceResponse,
|
||||
ProcessListResponse,
|
||||
SystemInfoResponse,
|
||||
VersionResponse,
|
||||
)
|
||||
from ledgrab.config import get_config, is_demo_mode
|
||||
@@ -278,6 +281,52 @@ async def get_running_processes(_: AuthRequired):
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/system/installed-apps",
|
||||
response_model=InstalledAppsResponse,
|
||||
tags=["Config"],
|
||||
)
|
||||
def get_installed_apps(_: AuthRequired):
|
||||
"""List launchable apps for the application-rule app picker (Android only).
|
||||
|
||||
Returns launchable apps (package + human label) on Android, where the
|
||||
foreground-app automation rule matches package names. Returns an empty list
|
||||
on desktop, where the process picker (``/system/processes``) is used instead.
|
||||
Sync ``def`` so FastAPI runs the (potentially blocking) bridge call in a
|
||||
thread pool.
|
||||
"""
|
||||
from ledgrab.core.automations import platform_detector as pd
|
||||
|
||||
try:
|
||||
apps = pd.list_installed_apps()
|
||||
items = [InstalledAppItem(package=a["package"], label=a["label"]) for a in apps]
|
||||
return InstalledAppsResponse(apps=items, count=len(items))
|
||||
except Exception as e:
|
||||
logger.error("Failed to list installed apps: %s", e, exc_info=True)
|
||||
raise HTTPException(status_code=500, detail="Internal server error")
|
||||
|
||||
|
||||
@router.get("/api/v1/system/info", response_model=SystemInfoResponse, tags=["Info"])
|
||||
def get_system_info(_: AuthRequired):
|
||||
"""Platform capability signal for the automation editor.
|
||||
|
||||
Tells the frontend whether the server is on Android (so the application-rule
|
||||
editor uses the launchable-app picker + package matching and surfaces the
|
||||
Usage-Access banner) vs desktop (process picker + process names), and whether
|
||||
Usage Access is currently granted. Sync ``def`` so the bridge call runs in a
|
||||
thread pool.
|
||||
"""
|
||||
from ledgrab.core.automations import platform_detector as pd
|
||||
from ledgrab.utils.platform import is_android
|
||||
|
||||
android = is_android()
|
||||
return SystemInfoResponse(
|
||||
is_android=android,
|
||||
app_match_kind="package" if android else "process",
|
||||
usage_access_granted=(pd.has_usage_access() if android else True),
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/system/performance",
|
||||
response_model=PerformanceResponse,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""System routes: MQTT, external URL, ADB, logs WebSocket, log level.
|
||||
"""System routes: external URL, shutdown action, ADB, logs WebSocket, log level.
|
||||
|
||||
Extracted from system.py to keep files under 800 lines.
|
||||
"""
|
||||
@@ -17,13 +17,10 @@ from ledgrab.api.schemas.system import (
|
||||
ExternalUrlResponse,
|
||||
LogLevelRequest,
|
||||
LogLevelResponse,
|
||||
MQTTSettingsRequest,
|
||||
MQTTSettingsResponse,
|
||||
ShutdownAction,
|
||||
ShutdownActionRequest,
|
||||
ShutdownActionResponse,
|
||||
)
|
||||
from ledgrab.config import get_config
|
||||
from ledgrab.storage.database import Database
|
||||
from ledgrab.utils import get_logger
|
||||
|
||||
@@ -32,85 +29,6 @@ logger = get_logger(__name__)
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MQTT settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _load_mqtt_settings(db: Database) -> dict:
|
||||
"""Load MQTT settings: YAML config defaults overridden by DB settings."""
|
||||
cfg = get_config()
|
||||
defaults = {
|
||||
"enabled": cfg.mqtt.enabled,
|
||||
"broker_host": cfg.mqtt.broker_host,
|
||||
"broker_port": cfg.mqtt.broker_port,
|
||||
"username": cfg.mqtt.username,
|
||||
"password": cfg.mqtt.password,
|
||||
"client_id": cfg.mqtt.client_id,
|
||||
"base_topic": cfg.mqtt.base_topic,
|
||||
}
|
||||
overrides = db.get_setting("mqtt")
|
||||
if overrides:
|
||||
defaults.update(overrides)
|
||||
return defaults
|
||||
|
||||
|
||||
@router.get(
|
||||
"/api/v1/system/mqtt/settings",
|
||||
response_model=MQTTSettingsResponse,
|
||||
tags=["System"],
|
||||
)
|
||||
async def get_mqtt_settings(_: AuthRequired, db: Database = Depends(get_database)):
|
||||
"""Get current MQTT broker settings. Password is masked."""
|
||||
s = _load_mqtt_settings(db)
|
||||
return MQTTSettingsResponse(
|
||||
enabled=s["enabled"],
|
||||
broker_host=s["broker_host"],
|
||||
broker_port=s["broker_port"],
|
||||
username=s["username"],
|
||||
password_set=bool(s.get("password")),
|
||||
client_id=s["client_id"],
|
||||
base_topic=s["base_topic"],
|
||||
)
|
||||
|
||||
|
||||
@router.put(
|
||||
"/api/v1/system/mqtt/settings",
|
||||
response_model=MQTTSettingsResponse,
|
||||
tags=["System"],
|
||||
)
|
||||
async def update_mqtt_settings(
|
||||
_: AuthRequired, body: MQTTSettingsRequest, db: Database = Depends(get_database)
|
||||
):
|
||||
"""Update MQTT broker settings. If password is empty string, the existing password is preserved."""
|
||||
current = _load_mqtt_settings(db)
|
||||
|
||||
# If caller sends an empty password, keep the existing one
|
||||
password = body.password if body.password else current.get("password", "")
|
||||
|
||||
new_settings = {
|
||||
"enabled": body.enabled,
|
||||
"broker_host": body.broker_host,
|
||||
"broker_port": body.broker_port,
|
||||
"username": body.username,
|
||||
"password": password,
|
||||
"client_id": body.client_id,
|
||||
"base_topic": body.base_topic,
|
||||
}
|
||||
db.set_setting("mqtt", new_settings)
|
||||
logger.info("MQTT settings updated")
|
||||
|
||||
return MQTTSettingsResponse(
|
||||
enabled=new_settings["enabled"],
|
||||
broker_host=new_settings["broker_host"],
|
||||
broker_port=new_settings["broker_port"],
|
||||
username=new_settings["username"],
|
||||
password_set=bool(new_settings["password"]),
|
||||
client_id=new_settings["client_id"],
|
||||
base_topic=new_settings["base_topic"],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# External URL setting
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -11,9 +11,21 @@ class RuleSchema(BaseModel):
|
||||
|
||||
rule_type: str = Field(description="Rule type discriminator (e.g. 'application')")
|
||||
# Application rule fields
|
||||
apps: List[str] | None = Field(None, description="Process names (for application rule)")
|
||||
apps: List[str] | None = Field(
|
||||
None,
|
||||
description=(
|
||||
"App identifiers for the application rule. Platform-specific and not "
|
||||
"portable: process names on Windows (e.g. 'chrome.exe'), package names "
|
||||
"on Android (e.g. 'com.android.chrome'). Matched case-insensitively."
|
||||
),
|
||||
)
|
||||
match_type: str | None = Field(
|
||||
None, description="'running' or 'topmost' (for application rule)"
|
||||
None,
|
||||
description=(
|
||||
"'running', 'topmost', 'fullscreen', or 'topmost_fullscreen' (application "
|
||||
"rule). On Android only the foreground app is detectable, so all values "
|
||||
"behave as 'foreground'."
|
||||
),
|
||||
)
|
||||
# Time-of-day rule fields
|
||||
start_time: str | None = Field(None, description="Start time HH:MM (for time_of_day rule)")
|
||||
|
||||
@@ -68,6 +68,42 @@ class ProcessListResponse(BaseModel):
|
||||
count: int = Field(description="Number of unique processes")
|
||||
|
||||
|
||||
class InstalledAppItem(BaseModel):
|
||||
"""A launchable Android app, for the automation app picker."""
|
||||
|
||||
package: str = Field(description="Android package name, e.g. 'com.netflix.mediaclient'")
|
||||
label: str = Field(description="Human-readable app label, e.g. 'Netflix'")
|
||||
|
||||
|
||||
class InstalledAppsResponse(BaseModel):
|
||||
"""Launchable apps for the application-rule picker (Android only; empty elsewhere)."""
|
||||
|
||||
apps: List[InstalledAppItem] = Field(description="Launchable apps, sorted by label")
|
||||
count: int = Field(description="Number of apps")
|
||||
|
||||
|
||||
class SystemInfoResponse(BaseModel):
|
||||
"""Platform capability signal for the frontend (automation editor).
|
||||
|
||||
Lets the application-rule editor choose the right app source and matching
|
||||
semantics per platform, and surface the Usage-Access permission state.
|
||||
"""
|
||||
|
||||
is_android: bool = Field(description="True when the server runs on Android (Chaquopy)")
|
||||
app_match_kind: Literal["process", "package"] = Field(
|
||||
description=(
|
||||
"What ApplicationRule.apps values represent: 'process' names on desktop, "
|
||||
"'package' names on Android."
|
||||
)
|
||||
)
|
||||
usage_access_granted: bool = Field(
|
||||
description=(
|
||||
"Android: whether PACKAGE_USAGE_STATS (Usage Access) is granted, gating "
|
||||
"foreground-app detection. Always True (not applicable) off-Android."
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class GpuInfo(BaseModel):
|
||||
"""GPU performance information."""
|
||||
|
||||
@@ -158,35 +194,6 @@ class BackupListResponse(BaseModel):
|
||||
count: int
|
||||
|
||||
|
||||
# ─── MQTT schemas ──────────────────────────────────────────────
|
||||
|
||||
|
||||
class MQTTSettingsResponse(BaseModel):
|
||||
"""MQTT broker settings response (password is masked)."""
|
||||
|
||||
enabled: bool = Field(description="Whether MQTT is enabled")
|
||||
broker_host: str = Field(description="MQTT broker hostname or IP")
|
||||
broker_port: int = Field(ge=1, le=65535, description="MQTT broker port")
|
||||
username: str = Field(description="MQTT username (empty = anonymous)")
|
||||
password_set: bool = Field(description="Whether a password is configured")
|
||||
client_id: str = Field(description="MQTT client ID")
|
||||
base_topic: str = Field(description="Base topic prefix")
|
||||
|
||||
|
||||
class MQTTSettingsRequest(BaseModel):
|
||||
"""MQTT broker settings update request."""
|
||||
|
||||
enabled: bool = Field(description="Whether MQTT is enabled")
|
||||
broker_host: str = Field(description="MQTT broker hostname or IP")
|
||||
broker_port: int = Field(ge=1, le=65535, description="MQTT broker port")
|
||||
username: str = Field(default="", description="MQTT username (empty = anonymous)")
|
||||
password: str = Field(
|
||||
default="", description="MQTT password (empty = keep existing if omitted)"
|
||||
)
|
||||
client_id: str = Field(default="ledgrab", description="MQTT client ID")
|
||||
base_topic: str = Field(default="ledgrab", description="Base topic prefix")
|
||||
|
||||
|
||||
# ─── External URL schema ───────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -6,12 +6,14 @@ Non-Windows: graceful degradation (returns empty results).
|
||||
|
||||
import asyncio
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from typing import Set
|
||||
|
||||
from ledgrab.utils import get_logger
|
||||
from ledgrab.utils.platform import is_android
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -21,6 +23,105 @@ if _IS_WINDOWS:
|
||||
import ctypes.wintypes
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Android ForegroundAppBridge interop — lazy + guarded (never at import time)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Android reports ``sys.platform == "linux"`` so ``_IS_WINDOWS`` is False there;
|
||||
# the foreground app is read via the Kotlin ``ForegroundAppBridge`` (UsageStats)
|
||||
# instead of Win32 ctypes. These module-level wrappers are the monkeypatch
|
||||
# surface used by tests (mirrors ``android_camera_engine``) — patch the module
|
||||
# function, not the live ``jclass`` object.
|
||||
|
||||
# Emit the "Usage Access not granted" warning only once per process so the ~1s
|
||||
# automation poll loop doesn't spam the log while access is missing.
|
||||
_warned_no_usage_access = False
|
||||
|
||||
|
||||
def _foreground_bridge():
|
||||
"""Return the Kotlin ``ForegroundAppBridge`` singleton, or None off-Android.
|
||||
|
||||
The ``from java import jclass`` import only resolves inside the Chaquopy
|
||||
runtime, so it must never run at module import time (this module is imported
|
||||
on desktop CI too). Mirrors ``android_camera_engine._camera_bridge()``.
|
||||
"""
|
||||
if not is_android():
|
||||
return None
|
||||
try:
|
||||
from java import jclass # type: ignore[import-not-found]
|
||||
except ImportError as exc:
|
||||
logger.debug("Chaquopy java interop not available: %s", exc)
|
||||
return None
|
||||
try:
|
||||
return jclass("com.ledgrab.android.ForegroundAppBridge").INSTANCE
|
||||
except Exception as exc: # pragma: no cover - Android-only path
|
||||
logger.debug("ForegroundAppBridge singleton unavailable: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def has_usage_access() -> bool:
|
||||
"""Whether Usage Access (PACKAGE_USAGE_STATS) is granted. False off-Android."""
|
||||
bridge = _foreground_bridge()
|
||||
if bridge is None:
|
||||
return False
|
||||
try:
|
||||
return bool(bridge.hasUsageAccess())
|
||||
except Exception as exc: # pragma: no cover - Android-only path
|
||||
logger.debug("ForegroundAppBridge.hasUsageAccess failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def get_foreground_package() -> str | None:
|
||||
"""Current foreground app package via the Kotlin bridge, or None.
|
||||
|
||||
None off-Android, when the bridge is unavailable, when Usage Access is
|
||||
missing, or when no foreground event is found in the trailing window.
|
||||
Monkeypatched in tests.
|
||||
"""
|
||||
bridge = _foreground_bridge()
|
||||
if bridge is None:
|
||||
return None
|
||||
try:
|
||||
pkg = bridge.getForegroundPackage()
|
||||
except Exception as exc: # pragma: no cover - Android-only path
|
||||
logger.warning("ForegroundAppBridge.getForegroundPackage failed: %s", exc)
|
||||
return None
|
||||
if pkg is None:
|
||||
return None
|
||||
s = str(pkg).strip()
|
||||
return s or None
|
||||
|
||||
|
||||
def list_installed_apps() -> list[dict]:
|
||||
"""Launchable apps via the Kotlin bridge: ``[{"package": .., "label": ..}]``.
|
||||
|
||||
Returns ``[]`` off-Android, when the bridge is unavailable, on error, or on
|
||||
invalid JSON. Sorted by label (the bridge sorts; order is preserved here).
|
||||
Monkeypatched in tests.
|
||||
"""
|
||||
bridge = _foreground_bridge()
|
||||
if bridge is None:
|
||||
return []
|
||||
try:
|
||||
raw = bridge.listLaunchableApps() # JSON array string
|
||||
except Exception as exc: # pragma: no cover - Android-only path
|
||||
logger.warning("ForegroundAppBridge.listLaunchableApps failed: %s", exc)
|
||||
return []
|
||||
try:
|
||||
parsed = json.loads(str(raw))
|
||||
except (ValueError, TypeError) as exc: # pragma: no cover - Android-only path
|
||||
logger.warning("ForegroundAppBridge.listLaunchableApps returned invalid JSON: %s", exc)
|
||||
return []
|
||||
apps: list[dict] = []
|
||||
for entry in parsed if isinstance(parsed, list) else []:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
pkg = entry.get("package")
|
||||
if not pkg:
|
||||
continue
|
||||
apps.append({"package": str(pkg), "label": str(entry.get("label") or pkg)})
|
||||
return apps
|
||||
|
||||
|
||||
class PlatformDetector:
|
||||
"""Detect running processes and the foreground window's process."""
|
||||
|
||||
@@ -215,6 +316,31 @@ class PlatformDetector:
|
||||
|
||||
# ---- Process detection ----
|
||||
|
||||
def _get_android_foreground(self) -> tuple:
|
||||
"""(package_lowercased, True) for the foreground app on Android.
|
||||
|
||||
Returns ``(None, False)`` when Usage Access is not granted (warned once)
|
||||
or no foreground app is found. ``is_fullscreen`` is reported True because
|
||||
a foreground TV app effectively covers the screen — so an Android rule's
|
||||
``topmost``/``topmost_fullscreen``/``fullscreen`` match types all behave
|
||||
as "this app is in front". Delegates to the module-level bridge wrappers
|
||||
(the monkeypatch surface used by tests).
|
||||
"""
|
||||
global _warned_no_usage_access
|
||||
if not has_usage_access():
|
||||
if not _warned_no_usage_access:
|
||||
logger.warning(
|
||||
"Android 'Application' automation rules need Usage Access "
|
||||
"(Settings > Usage access). Foreground-app rules will not match "
|
||||
"until it is granted."
|
||||
)
|
||||
_warned_no_usage_access = True
|
||||
return (None, False)
|
||||
pkg = get_foreground_package()
|
||||
if not pkg:
|
||||
return (None, False)
|
||||
return (pkg.lower(), True)
|
||||
|
||||
def _get_running_processes_sync(self) -> Set[str]:
|
||||
"""Get set of lowercase process names via Win32 EnumProcesses.
|
||||
|
||||
@@ -222,7 +348,14 @@ class PlatformDetector:
|
||||
which is ~300x faster than WMI (~8ms vs ~3s). System services
|
||||
running under protected accounts are not visible, but all
|
||||
user-facing applications are covered.
|
||||
|
||||
On Android there is no process enumeration API (getRunningTasks is
|
||||
restricted); the foreground app is reported as the sole "running" entry
|
||||
as a best-effort so ``match_type="running"`` rules still work.
|
||||
"""
|
||||
if is_android():
|
||||
pkg, _ = self._get_android_foreground()
|
||||
return {pkg} if pkg else set()
|
||||
if not _IS_WINDOWS:
|
||||
return set()
|
||||
|
||||
@@ -276,9 +409,13 @@ class PlatformDetector:
|
||||
def _get_topmost_process_sync(self) -> tuple:
|
||||
"""Get (process_name, is_fullscreen) of the foreground window.
|
||||
|
||||
Returns (None, False) when detection fails.
|
||||
On Android the "foreground window" is the foreground app package (read
|
||||
via the Kotlin ForegroundAppBridge); see ``_get_android_foreground``.
|
||||
Returns (None, False) when detection fails / Usage Access is missing.
|
||||
Blocking — call via executor.
|
||||
"""
|
||||
if is_android():
|
||||
return self._get_android_foreground()
|
||||
if not _IS_WINDOWS:
|
||||
return (None, False)
|
||||
|
||||
@@ -369,7 +506,13 @@ class PlatformDetector:
|
||||
|
||||
Enumerates all top-level windows and checks each for fullscreen.
|
||||
Returns process names (lowercase) whose window covers an entire monitor.
|
||||
|
||||
On Android the foreground app is treated as fullscreen, so it is the
|
||||
sole entry (best-effort, mirrors ``_get_running_processes_sync``).
|
||||
"""
|
||||
if is_android():
|
||||
pkg, _ = self._get_android_foreground()
|
||||
return {pkg} if pkg else set()
|
||||
if not _IS_WINDOWS:
|
||||
return set()
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ async def apply_scene_state(
|
||||
proc = processor_manager.get_processor(ts.target_id)
|
||||
if proc and proc.is_running:
|
||||
css_changed = "color_strip_source_id" in changed
|
||||
brightness_changed = "brightness" in changed
|
||||
brightness_changed = "brightness_value_source_id" in changed
|
||||
settings_changed = "fps" in changed
|
||||
if css_changed:
|
||||
target.sync_with_manager(
|
||||
|
||||
@@ -74,6 +74,19 @@
|
||||
font-size: 0.85rem;
|
||||
}
|
||||
|
||||
/* Android-only: shown in the application rule when Usage Access is missing,
|
||||
so the foreground-app rule can't fire until the user grants it on the TV. */
|
||||
.rule-usage-warning {
|
||||
margin-bottom: 10px;
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-size: 0.85rem;
|
||||
line-height: 1.35;
|
||||
color: var(--warning-color, #ff9800);
|
||||
background: color-mix(in srgb, var(--warning-color, #ff9800) 12%, transparent);
|
||||
border: 1px solid color-mix(in srgb, var(--warning-color, #ff9800) 35%, transparent);
|
||||
}
|
||||
|
||||
.btn-remove-rule {
|
||||
background: none;
|
||||
border: none;
|
||||
|
||||
@@ -2,13 +2,19 @@
|
||||
* Command-palette style name picker — reusable UI for browsing a list of
|
||||
* names fetched from any API endpoint. Mirrors the EntityPalette pattern.
|
||||
*
|
||||
* Two concrete pickers are exported:
|
||||
* Three concrete pickers are exported:
|
||||
*
|
||||
* - **ProcessPalette** — picks from running OS processes (`/system/processes`)
|
||||
* - **NotificationAppPalette** — picks from OS notification history apps
|
||||
* - **AppPalette** — picks from Android launchable apps (`/system/installed-apps`),
|
||||
* displaying the human label but inserting the package name
|
||||
*
|
||||
* Both support single-select (returns one value) and multi-select (appends to
|
||||
* a textarea).
|
||||
* Items may be plain strings (display == stored value) or `{ value, label }`
|
||||
* pairs (display the label, store the value — used by AppPalette so the rule
|
||||
* stores the package name while the user sees "Netflix").
|
||||
*
|
||||
* All support single-select (returns one value) and multi-select (appends the
|
||||
* value to a textarea).
|
||||
*
|
||||
* Usage:
|
||||
*
|
||||
@@ -29,8 +35,16 @@ import { ICON_SEARCH } from './icons.ts';
|
||||
|
||||
/* ─── types ────────────────────────────────────────────────── */
|
||||
|
||||
interface PaletteItem {
|
||||
name: string;
|
||||
/** An item with a display label distinct from its stored value. */
|
||||
interface AppItem {
|
||||
value: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
/** Raw items a fetcher may return: bare strings or labelled pairs. */
|
||||
type RawItem = string | AppItem;
|
||||
|
||||
interface PaletteEntry extends AppItem {
|
||||
added: boolean;
|
||||
}
|
||||
|
||||
@@ -44,7 +58,9 @@ interface PickMultiOpts {
|
||||
placeholder?: string;
|
||||
}
|
||||
|
||||
type FetchItemsFn = () => Promise<string[]>;
|
||||
type FetchItemsFn = () => Promise<RawItem[]>;
|
||||
|
||||
const DEFAULT_EMPTY_KEY = 'automations.condition.application.no_processes';
|
||||
|
||||
/* ─── generic NamePalette (shared logic) ───────────────────── */
|
||||
|
||||
@@ -53,19 +69,21 @@ class NamePalette {
|
||||
private _input: HTMLInputElement;
|
||||
private _list: HTMLDivElement;
|
||||
private _fetchItems: FetchItemsFn;
|
||||
private _emptyKey: string;
|
||||
|
||||
private _resolveSingle: ((v: string | undefined) => void) | null = null;
|
||||
private _multiTextarea: HTMLTextAreaElement | null = null;
|
||||
|
||||
private _items: string[] = [];
|
||||
private _items: AppItem[] = [];
|
||||
private _existing: Set<string> = new Set();
|
||||
private _filtered: PaletteItem[] = [];
|
||||
private _filtered: PaletteEntry[] = [];
|
||||
private _highlightIdx = 0;
|
||||
private _currentValue: string | undefined;
|
||||
private _isMulti = false;
|
||||
|
||||
constructor(fetchItems: FetchItemsFn) {
|
||||
constructor(fetchItems: FetchItemsFn, emptyKey: string = DEFAULT_EMPTY_KEY) {
|
||||
this._fetchItems = fetchItems;
|
||||
this._emptyKey = emptyKey;
|
||||
|
||||
this._overlay = document.createElement('div');
|
||||
this._overlay.className = 'entity-palette-overlay process-palette-overlay';
|
||||
@@ -107,14 +125,20 @@ class NamePalette {
|
||||
this._isMulti = true;
|
||||
this._multiTextarea = opts.textarea;
|
||||
this._resolveSingle = resolve as any;
|
||||
this._existing = new Set(
|
||||
opts.textarea.value.split('\n').map(s => s.trim().toLowerCase()).filter(Boolean),
|
||||
);
|
||||
this._existing = this._textareaValues(opts.textarea);
|
||||
this._currentValue = undefined;
|
||||
this._open(opts.placeholder);
|
||||
});
|
||||
}
|
||||
|
||||
private _textareaValues(ta: HTMLTextAreaElement): Set<string> {
|
||||
return new Set(ta.value.split('\n').map(s => s.trim().toLowerCase()).filter(Boolean));
|
||||
}
|
||||
|
||||
private _normalize(raw: RawItem[]): AppItem[] {
|
||||
return raw.map(r => (typeof r === 'string' ? { value: r, label: r } : r));
|
||||
}
|
||||
|
||||
private async _open(placeholder?: string) {
|
||||
this._input.placeholder = placeholder || '';
|
||||
this._input.value = '';
|
||||
@@ -123,15 +147,13 @@ class NamePalette {
|
||||
requestAnimationFrame(() => this._input.focus());
|
||||
|
||||
try {
|
||||
this._items = await this._fetchItems();
|
||||
this._items = this._normalize(await this._fetchItems());
|
||||
} catch {
|
||||
this._items = [];
|
||||
}
|
||||
|
||||
if (this._isMulti) {
|
||||
this._existing = new Set(
|
||||
this._multiTextarea!.value.split('\n').map(s => s.trim().toLowerCase()).filter(Boolean),
|
||||
);
|
||||
this._existing = this._textareaValues(this._multiTextarea!);
|
||||
}
|
||||
|
||||
this._filter();
|
||||
@@ -142,14 +164,11 @@ class NamePalette {
|
||||
private _filter() {
|
||||
const q = this._input.value.toLowerCase().trim();
|
||||
this._filtered = this._items
|
||||
.filter(p => !q || p.toLowerCase().includes(q))
|
||||
.map(p => ({
|
||||
name: p,
|
||||
added: this._existing.has(p.toLowerCase()),
|
||||
}));
|
||||
.filter(p => !q || p.label.toLowerCase().includes(q) || p.value.toLowerCase().includes(q))
|
||||
.map(p => ({ ...p, added: this._existing.has(p.value.toLowerCase()) }));
|
||||
|
||||
this._highlightIdx = this._filtered.findIndex(
|
||||
i => i.name.toLowerCase() === (this._currentValue || '').toLowerCase(),
|
||||
i => i.value.toLowerCase() === (this._currentValue || '').toLowerCase(),
|
||||
);
|
||||
if (this._highlightIdx === -1) this._highlightIdx = 0;
|
||||
this._render();
|
||||
@@ -158,9 +177,7 @@ class NamePalette {
|
||||
private _render() {
|
||||
if (this._filtered.length === 0) {
|
||||
this._list.innerHTML = `<div class="entity-palette-empty">${
|
||||
this._items.length === 0
|
||||
? t('automations.condition.application.no_processes')
|
||||
: '—'
|
||||
this._items.length === 0 ? t(this._emptyKey) : '—'
|
||||
}</div>`;
|
||||
return;
|
||||
}
|
||||
@@ -170,12 +187,21 @@ class NamePalette {
|
||||
'entity-palette-item',
|
||||
i === this._highlightIdx ? 'ep-highlight' : '',
|
||||
item.added ? 'ep-current' : '',
|
||||
item.name.toLowerCase() === (this._currentValue || '').toLowerCase() ? 'ep-current' : '',
|
||||
item.value.toLowerCase() === (this._currentValue || '').toLowerCase() ? 'ep-current' : '',
|
||||
].filter(Boolean).join(' ');
|
||||
|
||||
// When the label differs from the stored value (e.g. "Netflix" vs
|
||||
// "com.netflix.mediaclient"), show the value as a secondary line so
|
||||
// users can see exactly what gets matched. Otherwise fall back to the
|
||||
// ✓ added-marker.
|
||||
const showValue = item.label !== item.value;
|
||||
const trailing = showValue
|
||||
? `<span class="ep-item-desc">${escapeHtml(item.value)}</span>`
|
||||
: (item.added ? '<span class="ep-item-desc">✓</span>' : '');
|
||||
|
||||
return `<div class="${cls}" data-idx="${i}">
|
||||
<span class="ep-item-label">${escapeHtml(item.name)}</span>
|
||||
${item.added ? '<span class="ep-item-desc">\u2713</span>' : ''}
|
||||
<span class="ep-item-label">${escapeHtml(item.label)}</span>
|
||||
${trailing}
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
@@ -192,19 +218,19 @@ class NamePalette {
|
||||
|
||||
/* ── selection ──────────────────────────────────────────── */
|
||||
|
||||
private _selectItem(item: PaletteItem) {
|
||||
private _selectItem(item: PaletteEntry) {
|
||||
if (this._isMulti) {
|
||||
if (!item.added) {
|
||||
const ta = this._multiTextarea!;
|
||||
const cur = ta.value.trim();
|
||||
ta.value = cur ? cur + '\n' + item.name : item.name;
|
||||
this._existing.add(item.name.toLowerCase());
|
||||
ta.value = cur ? cur + '\n' + item.value : item.value;
|
||||
this._existing.add(item.value.toLowerCase());
|
||||
item.added = true;
|
||||
this._render();
|
||||
}
|
||||
} else {
|
||||
this._overlay.classList.remove('open');
|
||||
if (this._resolveSingle) this._resolveSingle(item.name);
|
||||
if (this._resolveSingle) this._resolveSingle(item.value);
|
||||
this._resolveSingle = null;
|
||||
}
|
||||
}
|
||||
@@ -269,6 +295,17 @@ async function _fetchNotificationApps(): Promise<string[]> {
|
||||
return Array.from(seen.values()).sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
async function _fetchInstalledApps(): Promise<AppItem[]> {
|
||||
try {
|
||||
const data = await apiGet<{ apps?: Array<{ package: string; label: string }> }>(
|
||||
'/system/installed-apps',
|
||||
);
|
||||
return (data.apps || []).map(a => ({ value: a.package, label: a.label || a.package }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/* ─── ProcessPalette (running processes) ───────────────────── */
|
||||
|
||||
let _processInst: NamePalette | null = null;
|
||||
@@ -301,6 +338,22 @@ export class NotificationAppPalette {
|
||||
}
|
||||
}
|
||||
|
||||
/* ─── AppPalette (Android launchable apps) ─────────────────── */
|
||||
|
||||
let _appInst: NamePalette | null = null;
|
||||
|
||||
export class AppPalette {
|
||||
static pick(opts: PickOpts): Promise<string | undefined> {
|
||||
if (!_appInst) _appInst = new NamePalette(_fetchInstalledApps, 'automations.rule.application.no_apps');
|
||||
return _appInst.pickSingle(opts);
|
||||
}
|
||||
|
||||
static pickMulti(opts: PickMultiOpts): Promise<void> {
|
||||
if (!_appInst) _appInst = new NamePalette(_fetchInstalledApps, 'automations.rule.application.no_apps');
|
||||
return _appInst.pickMulti(opts);
|
||||
}
|
||||
}
|
||||
|
||||
/* ─── drop-in replacement for the old attachProcessPicker ─── */
|
||||
|
||||
/**
|
||||
@@ -334,3 +387,19 @@ export function attachNotificationAppPicker(containerEl: HTMLElement, textareaEl
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wire up a `.btn-browse-apps` button to open the Android launchable-app palette
|
||||
* (multi-select, feeding package names into a textarea while showing labels).
|
||||
*/
|
||||
export function attachAppPicker(containerEl: HTMLElement, textareaEl: HTMLTextAreaElement): void {
|
||||
const browseBtn = containerEl.querySelector('.btn-browse-apps');
|
||||
if (!browseBtn) return;
|
||||
|
||||
browseBtn.addEventListener('click', () => {
|
||||
AppPalette.pickMulti({
|
||||
textarea: textareaEl,
|
||||
placeholder: t('automations.rule.application.search_apps') || 'Filter apps…',
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ import { getBaseOrigin } from './settings.ts';
|
||||
import { IconSelect } from '../core/icon-select.ts';
|
||||
import { EntitySelect } from '../core/entity-palette.ts';
|
||||
import { enhanceMiniSelects } from '../core/mini-select.ts';
|
||||
import { attachProcessPicker } from '../core/process-picker.ts';
|
||||
import { attachProcessPicker, attachAppPicker } from '../core/process-picker.ts';
|
||||
import { TreeNav } from '../core/tree-nav.ts';
|
||||
import { csScenes, createSceneCard, initScenePresetDelegation } from './scene-presets.ts';
|
||||
import type { Automation, RuleType } from '../types.ts';
|
||||
@@ -215,6 +215,28 @@ document.addEventListener('server:automation_state_changed', () => {
|
||||
if (apiKey && isActiveTab('automations')) loadAutomations();
|
||||
});
|
||||
|
||||
/** Platform capability signal from `/system/info` — drives the application-rule
|
||||
* editor (process picker + match types on desktop vs. app picker + foreground-only
|
||||
* on Android) and the Usage-Access banner. Fetched once and cached. */
|
||||
interface PlatformInfo {
|
||||
is_android: boolean;
|
||||
app_match_kind: 'process' | 'package';
|
||||
usage_access_granted: boolean;
|
||||
}
|
||||
|
||||
let _platformInfo: PlatformInfo | null = null;
|
||||
|
||||
async function ensurePlatformInfo(): Promise<PlatformInfo> {
|
||||
if (_platformInfo) return _platformInfo;
|
||||
try {
|
||||
_platformInfo = await apiGet<PlatformInfo>('/system/info');
|
||||
} catch {
|
||||
// Default to desktop semantics if the signal can't be fetched.
|
||||
_platformInfo = { is_android: false, app_match_kind: 'process', usage_access_granted: true };
|
||||
}
|
||||
return _platformInfo;
|
||||
}
|
||||
|
||||
export async function loadAutomations() {
|
||||
if (_automationsLoading) return;
|
||||
set_automationsLoading(true);
|
||||
@@ -222,6 +244,10 @@ export async function loadAutomations() {
|
||||
if (!container) { set_automationsLoading(false); return; }
|
||||
if (!csAutomations.isMounted()) setTabRefreshing('automations-content', true);
|
||||
|
||||
// Prime the platform signal so the editor renders the right app source +
|
||||
// match semantics without an async hop when a rule row is expanded.
|
||||
void ensurePlatformInfo();
|
||||
|
||||
try {
|
||||
const [automations, scenes] = await Promise.all([
|
||||
automationsCacheObj.fetch(),
|
||||
@@ -559,6 +585,11 @@ export async function openAutomationEditor(automationId?: any, cloneData?: any)
|
||||
errorEl.style.display = 'none';
|
||||
ruleList!.innerHTML = '';
|
||||
|
||||
// Ensure the platform signal is loaded before rendering rule rows so the
|
||||
// application rule picks the right app source + match semantics. The
|
||||
// automations tab primes this, but the graph editor opens this directly.
|
||||
await ensurePlatformInfo();
|
||||
|
||||
_ensureRuleLogicIconSelect();
|
||||
_ensureDeactivationModeIconSelect();
|
||||
|
||||
@@ -1129,6 +1160,33 @@ function _renderWebhookFields(container: HTMLElement, data: any): void {
|
||||
|
||||
function _renderApplicationFields(container: HTMLElement, data: any): void {
|
||||
const appsValue = (data.apps || []).join('\n');
|
||||
|
||||
// On Android there is exactly one obtainable signal — the foreground app —
|
||||
// so the match-type selector is hidden (match_type is forced to "topmost" by
|
||||
// the collector) and the app list comes from launchable apps (package names)
|
||||
// rather than running processes (process names).
|
||||
if (_platformInfo?.is_android) {
|
||||
const banner = _platformInfo.usage_access_granted
|
||||
? ''
|
||||
: `<div class="rule-usage-warning">${t('automations.rule.application.usage_access_required')}</div>`;
|
||||
container.innerHTML = `
|
||||
<div class="rule-fields">
|
||||
${banner}
|
||||
<div class="rule-field">
|
||||
<div class="rule-apps-header">
|
||||
<label>${t('automations.rule.application.apps')}</label>
|
||||
<button type="button" class="btn btn-icon btn-secondary btn-browse-apps" title="${t('automations.rule.application.browse')}">${ICON_SEARCH}</button>
|
||||
</div>
|
||||
<textarea class="rule-apps" rows="3" placeholder="com.netflix.mediaclient com.android.chrome">${escapeHtml(appsValue)}</textarea>
|
||||
<small class="rule-hint-desc">${t('automations.rule.application.apps.hint_android')}</small>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
const textarea = container.querySelector('.rule-apps') as HTMLTextAreaElement;
|
||||
attachAppPicker(container, textarea);
|
||||
return;
|
||||
}
|
||||
|
||||
const matchType = data.match_type || 'running';
|
||||
container.innerHTML = `
|
||||
<div class="rule-fields">
|
||||
@@ -1299,7 +1357,10 @@ const RULE_COLLECTORS: Record<RuleType, RuleCollector> = {
|
||||
return r;
|
||||
},
|
||||
application: (row) => {
|
||||
const matchType = (row.querySelector('.rule-match-type') as HTMLSelectElement).value;
|
||||
// On Android the match-type selector is hidden (only the foreground app is
|
||||
// detectable), so default to "topmost" when the select isn't present.
|
||||
const matchSel = row.querySelector('.rule-match-type') as HTMLSelectElement | null;
|
||||
const matchType = matchSel ? matchSel.value : 'topmost';
|
||||
const appsText = (row.querySelector('.rule-apps') as HTMLTextAreaElement).value.trim();
|
||||
const apps = appsText ? appsText.split('\n').map(a => a.trim()).filter(Boolean) : [];
|
||||
return { rule_type: 'application', apps, match_type: matchType };
|
||||
|
||||
@@ -1226,6 +1226,10 @@
|
||||
"automations.rule.application.match_type.topmost_fullscreen.desc": "Foreground + fullscreen",
|
||||
"automations.rule.application.match_type.fullscreen": "Fullscreen",
|
||||
"automations.rule.application.match_type.fullscreen.desc": "Any fullscreen app",
|
||||
"automations.rule.application.apps.hint_android": "Package names, one per line (e.g. com.netflix.mediaclient)",
|
||||
"automations.rule.application.search_apps": "Filter apps...",
|
||||
"automations.rule.application.no_apps": "No apps found",
|
||||
"automations.rule.application.usage_access_required": "Needs Usage Access. On your LedGrab TV, open the app and tap 'Grant usage access'.",
|
||||
"automations.rule.time_of_day": "Time of Day",
|
||||
"automations.rule.time_of_day.desc": "Time range",
|
||||
"automations.rule.time_of_day.start_time": "Start Time:",
|
||||
|
||||
@@ -1260,6 +1260,10 @@
|
||||
"automations.rule.application.match_type.topmost_fullscreen.desc": "В фокусе + полный экран",
|
||||
"automations.rule.application.match_type.fullscreen": "Полный экран",
|
||||
"automations.rule.application.match_type.fullscreen.desc": "Любое полноэкранное",
|
||||
"automations.rule.application.apps.hint_android": "Имена пакетов, по одному в строке (напр. com.netflix.mediaclient)",
|
||||
"automations.rule.application.search_apps": "Поиск приложений...",
|
||||
"automations.rule.application.no_apps": "Приложения не найдены",
|
||||
"automations.rule.application.usage_access_required": "Требуется доступ к статистике использования. Откройте LedGrab на телевизоре и нажмите «Разрешить доступ к статистике использования».",
|
||||
"automations.rule.time_of_day": "Время суток",
|
||||
"automations.rule.time_of_day.desc": "Диапазон времени",
|
||||
"automations.rule.time_of_day.start_time": "Время начала:",
|
||||
|
||||
@@ -1256,6 +1256,10 @@
|
||||
"automations.rule.application.match_type.topmost_fullscreen.desc": "前台 + 全屏",
|
||||
"automations.rule.application.match_type.fullscreen": "全屏",
|
||||
"automations.rule.application.match_type.fullscreen.desc": "任意全屏应用",
|
||||
"automations.rule.application.apps.hint_android": "包名，每行一个（例如 com.netflix.mediaclient）",
|
||||
"automations.rule.application.search_apps": "筛选应用…",
|
||||
"automations.rule.application.no_apps": "未找到应用",
|
||||
"automations.rule.application.usage_access_required": "需要使用情况访问权限。在您的 LedGrab 电视上打开应用并点按「授予使用情况访问权限」。",
|
||||
"automations.rule.time_of_day": "时段",
|
||||
"automations.rule.time_of_day.desc": "时间范围",
|
||||
"automations.rule.time_of_day.start_time": "开始时间：",
|
||||
|
||||
@@ -30,11 +30,24 @@ class Rule:
|
||||
|
||||
@dataclass
|
||||
class ApplicationRule(Rule):
|
||||
"""Activate when specified applications are running or topmost."""
|
||||
"""Activate when specified applications are running or topmost.
|
||||
|
||||
``apps`` values are platform-specific and NOT portable across OSes:
|
||||
on Windows they are **process names** (e.g. ``chrome.exe``); on Android
|
||||
they are **package names** (e.g. ``com.android.chrome``). Matching is
|
||||
exact and case-insensitive. The automation editor sources values from the
|
||||
right place per platform (running processes on desktop, launchable apps on
|
||||
Android), so a rule authored on one OS will simply not match on another.
|
||||
|
||||
``match_type`` is honoured on Windows for all four values below. On Android
|
||||
only the foreground app is obtainable, so every match type collapses to
|
||||
"this app is in the foreground" and the editor hides the selector.
|
||||
"""
|
||||
|
||||
rule_type: str = "application"
|
||||
apps: List[str] = field(default_factory=list)
|
||||
match_type: str = "running" # "running" | "topmost"
|
||||
# "running" | "topmost" | "fullscreen" | "topmost_fullscreen"
|
||||
match_type: str = "running"
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
d = super().to_dict()
|
||||
|
||||
@@ -92,3 +92,57 @@ class TestRootEndpoint:
|
||||
resp = client.get("/")
|
||||
assert resp.status_code == 200
|
||||
assert "text/html" in resp.headers["content-type"]
|
||||
|
||||
|
||||
class TestInstalledAppsEndpoint:
|
||||
def test_requires_auth(self, client):
|
||||
resp = client.get("/api/v1/system/installed-apps")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_empty_off_android(self, client):
|
||||
"""Desktop test host: is_android() is False, so the bridge wrapper
|
||||
short-circuits to an empty list."""
|
||||
resp = client.get("/api/v1/system/installed-apps", headers=_auth_headers())
|
||||
assert resp.status_code == 200
|
||||
assert resp.json() == {"apps": [], "count": 0}
|
||||
|
||||
def test_returns_apps_when_available(self, client, monkeypatch):
|
||||
from ledgrab.core.automations import platform_detector as pd
|
||||
|
||||
monkeypatch.setattr(
|
||||
pd,
|
||||
"list_installed_apps",
|
||||
lambda: [{"package": "com.netflix.mediaclient", "label": "Netflix"}],
|
||||
)
|
||||
resp = client.get("/api/v1/system/installed-apps", headers=_auth_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["count"] == 1
|
||||
assert data["apps"][0] == {"package": "com.netflix.mediaclient", "label": "Netflix"}
|
||||
|
||||
|
||||
class TestSystemInfoEndpoint:
|
||||
def test_requires_auth(self, client):
|
||||
resp = client.get("/api/v1/system/info")
|
||||
assert resp.status_code == 401
|
||||
|
||||
def test_desktop_signal(self, client):
|
||||
resp = client.get("/api/v1/system/info", headers=_auth_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["is_android"] is False
|
||||
assert data["app_match_kind"] == "process"
|
||||
assert data["usage_access_granted"] is True
|
||||
|
||||
def test_android_signal(self, client, monkeypatch):
|
||||
import ledgrab.utils.platform as plat
|
||||
from ledgrab.core.automations import platform_detector as pd
|
||||
|
||||
monkeypatch.setattr(plat, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: False)
|
||||
resp = client.get("/api/v1/system/info", headers=_auth_headers())
|
||||
assert resp.status_code == 200
|
||||
data = resp.json()
|
||||
assert data["is_android"] is True
|
||||
assert data["app_match_kind"] == "package"
|
||||
assert data["usage_access_granted"] is False
|
||||
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Tests for Android foreground-app detection in PlatformDetector.
|
||||
|
||||
These run on desktop CI (no Android device needed): ``is_android`` and the
|
||||
Kotlin-bridge wrappers (``has_usage_access`` / ``get_foreground_package``) are
|
||||
monkeypatched, exactly as the Kotlin ``ForegroundAppBridge`` would drive them on
|
||||
device. The critical invariant under test is that the Android branch runs *ahead
|
||||
of* the import-time ``_IS_WINDOWS`` guard, and that the Windows/desktop paths are
|
||||
left untouched.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from ledgrab.core.automations import platform_detector as pd
|
||||
from ledgrab.core.automations.platform_detector import PlatformDetector
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def detector(monkeypatch):
|
||||
"""A PlatformDetector with the Windows display-power listener stubbed out.
|
||||
|
||||
``__init__`` otherwise spawns a thread that registers a global window class +
|
||||
runs a ctypes message pump — irrelevant here and noisy when many detectors are
|
||||
constructed in one process.
|
||||
"""
|
||||
monkeypatch.setattr(PlatformDetector, "_display_power_listener", lambda self: None)
|
||||
return PlatformDetector()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_warn():
|
||||
"""Reset the process-global warn-once flag around every test."""
|
||||
pd._warned_no_usage_access = False
|
||||
yield
|
||||
pd._warned_no_usage_access = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# topmost (foreground) detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_topmost_android_returns_lowercased_foreground_package(detector, monkeypatch):
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: True)
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: "com.Netflix.MediaClient")
|
||||
|
||||
assert detector._get_topmost_process_sync() == ("com.netflix.mediaclient", True)
|
||||
|
||||
|
||||
def test_topmost_android_no_access_returns_none_and_warns_once(detector, monkeypatch):
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: False)
|
||||
fg_calls = []
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: fg_calls.append(1) or "x")
|
||||
warns = []
|
||||
monkeypatch.setattr(pd.logger, "warning", lambda *a, **k: warns.append(a))
|
||||
|
||||
assert detector._get_topmost_process_sync() == (None, False)
|
||||
assert detector._get_topmost_process_sync() == (None, False)
|
||||
|
||||
# Foreground is never queried when access is missing; warned exactly once.
|
||||
assert fg_calls == []
|
||||
assert len(warns) == 1
|
||||
assert pd._warned_no_usage_access is True
|
||||
|
||||
|
||||
def test_topmost_android_no_foreground_event_returns_none(detector, monkeypatch):
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: True)
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: None)
|
||||
|
||||
assert detector._get_topmost_process_sync() == (None, False)
|
||||
|
||||
|
||||
def test_android_branch_precedes_windows_guard(detector, monkeypatch):
|
||||
"""Even with _IS_WINDOWS True, is_android() must win.
|
||||
|
||||
Proves the Android branch sits ahead of the ``if not _IS_WINDOWS`` early
|
||||
return and never falls through to the Win32 ctypes path (the plan-review
|
||||
critical gap: a naive wiring would no-op behind the Windows guard).
|
||||
"""
|
||||
monkeypatch.setattr(pd, "_IS_WINDOWS", True)
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: True)
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: "com.App.X")
|
||||
|
||||
assert detector._get_topmost_process_sync() == ("com.app.x", True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# running / fullscreen best-effort (foreground app as the sole entry)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_running_and_fullscreen_android_return_foreground_set(detector, monkeypatch):
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: True)
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: "com.App.Y")
|
||||
|
||||
assert detector._get_running_processes_sync() == {"com.app.y"}
|
||||
assert detector._get_fullscreen_processes_sync() == {"com.app.y"}
|
||||
|
||||
|
||||
def test_running_and_fullscreen_android_empty_without_access(detector, monkeypatch):
|
||||
monkeypatch.setattr(pd, "is_android", lambda: True)
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: False)
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: "x")
|
||||
|
||||
assert detector._get_running_processes_sync() == set()
|
||||
assert detector._get_fullscreen_processes_sync() == set()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# desktop paths untouched
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_non_android_non_windows_skips_bridge(detector, monkeypatch):
|
||||
"""Desktop Linux/mac: no Android branch, no Win32 path, empty results, and
|
||||
the bridge wrappers are never consulted."""
|
||||
monkeypatch.setattr(pd, "_IS_WINDOWS", False)
|
||||
monkeypatch.setattr(pd, "is_android", lambda: False)
|
||||
calls = []
|
||||
monkeypatch.setattr(pd, "get_foreground_package", lambda: calls.append("fg"))
|
||||
monkeypatch.setattr(pd, "has_usage_access", lambda: calls.append("acc") or True)
|
||||
|
||||
assert detector._get_topmost_process_sync() == (None, False)
|
||||
assert detector._get_running_processes_sync() == set()
|
||||
assert detector._get_fullscreen_processes_sync() == set()
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_wrappers_return_safe_defaults_off_android(monkeypatch):
|
||||
"""is_android() False short-circuits the bridge accessor to None, so the
|
||||
public wrappers return safe defaults without any java interop."""
|
||||
monkeypatch.setattr(pd, "is_android", lambda: False)
|
||||
|
||||
assert pd._foreground_bridge() is None
|
||||
assert pd.has_usage_access() is False
|
||||
assert pd.get_foreground_package() is None
|
||||
assert pd.list_installed_apps() == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# bridge-response parsing wrappers (fed via a fake bridge object)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeBridge:
|
||||
"""Stand-in for the Kotlin ForegroundAppBridge singleton."""
|
||||
|
||||
def __init__(self, fg=None, apps_json=None):
|
||||
self._fg = fg
|
||||
self._apps_json = apps_json
|
||||
|
||||
def getForegroundPackage(self):
|
||||
return self._fg
|
||||
|
||||
def listLaunchableApps(self):
|
||||
return self._apps_json
|
||||
|
||||
|
||||
def test_get_foreground_package_strips_whitespace(monkeypatch):
|
||||
# Stripped but NOT lowercased — the caller (_get_android_foreground) lowercases.
|
||||
monkeypatch.setattr(pd, "_foreground_bridge", lambda: _FakeBridge(fg=" com.App.X "))
|
||||
assert pd.get_foreground_package() == "com.App.X"
|
||||
|
||||
|
||||
def test_get_foreground_package_blank_returns_none(monkeypatch):
|
||||
monkeypatch.setattr(pd, "_foreground_bridge", lambda: _FakeBridge(fg=" "))
|
||||
assert pd.get_foreground_package() is None
|
||||
|
||||
|
||||
def test_list_installed_apps_parses_and_filters(monkeypatch):
|
||||
import json
|
||||
|
||||
payload = json.dumps(
|
||||
[
|
||||
{"package": "com.a", "label": "A"},
|
||||
{"package": "com.b", "label": ""}, # blank label -> falls back to package
|
||||
{"label": "no package"}, # skipped: no package
|
||||
"not a dict", # skipped: not an object
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(pd, "_foreground_bridge", lambda: _FakeBridge(apps_json=payload))
|
||||
assert pd.list_installed_apps() == [
|
||||
{"package": "com.a", "label": "A"},
|
||||
{"package": "com.b", "label": "com.b"},
|
||||
]
|
||||
|
||||
|
||||
def test_list_installed_apps_invalid_json_returns_empty(monkeypatch):
|
||||
monkeypatch.setattr(pd, "_foreground_bridge", lambda: _FakeBridge(apps_json="not json{"))
|
||||
assert pd.list_installed_apps() == []
|
||||
Reference in New Issue
Block a user