alexei.dolgolyov/ledgrab
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 14 Wiki Activity

feat: aggregated snapshot + wiring-graph APIs, MQTT device brokers

Browse Source 
Backend
- snapshot: GET /api/v1/snapshot aggregates targets, devices, sources,
  presets and system into one payload for the HA coordinator, collapsing
  the prior ~2N+M request fan-out; per-section ?include= gating.
- graph: GET /api/v1/graph{,/schema,/dependents} backed by a pure,
  unit-tested graph_schema engine — one authoritative connectable-field
  registry so the editor no longer hard-codes topology in two places.
- devices: thread mqtt_source_id through DeviceCreate/Update/Response and
  the routes for multi-broker MQTT; shared validate_mqtt_source_exists
  (_mqtt_validation.py) reused by device + output-target routes; stop
  update_device masking intentional 4xx as 500.
- shutdown: bound uvicorn graceful-shutdown via GRACEFUL_SHUTDOWN_TIMEOUT
  (shared by __main__, android_entry, demo) so a lingering events WebSocket
  can't strand LED targets or block process exit.
- access log: structured _access_log middleware attributing each request to
  its authenticated token label (never the secret); uvicorn access_log off.

Frontend
- graph editor: generic schema-driven port/edge rendering, layout and
  connection handling; service-worker refresh.
- device modals: MQTT broker EntitySelect for device_type=mqtt in add-device
  and settings, wired into load/save/validate/dirty-check/clone.
- i18n: en/ru/zh keys.

Tests: graph routes + schema, snapshot routes, access log, mqtt_source_id
device regressions, bounded-shutdown entrypoint. 1614 passed.
This commit is contained in:
Admin
2026-05-28 22:51:04 +03:00
parent b83a72e63f
commit a5effba553
37 changed files with 3068 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/ledgrab/api/graph_schema.py
+501
View File
@@ -0,0 +1,501 @@
"""Authoritative wiring-graph schema and topology engine.
This module is the single source of truth for **which reference fields connect
which entity kinds**. The frontend graph editor historically hard-coded the same
information in two places (``graph-connections.ts`` ``CONNECTION_MAP`` and
``graph-layout.ts`` ``buildGraph``); the ``GET /api/v1/graph/schema`` endpoint
now serves this registry so the client can render ports and edges generically
and the two never drift.
This registry is a *superset* of the current frontend ``buildGraph``: it also
declares real references that ``buildGraph`` does not yet draw (e.g.
``value_source.value_source_id`` chaining and ``value_source.color_strip_source_id``).
The backend is authoritative; the client is expected to converge on it.
Everything in this module is pure (operates on plain dicts), so the topology
build, dependency lookup, cycle and dangling-reference detection are all unit
testable without booting the app or any store.
Field-path grammar (the ``field`` of a :class:`ConnectionField`):
* ``"device_id"`` — a top-level string id.
* ``"brightness.source_id"`` — a nested object; ``brightness`` may be a
plain number (unbound :class:`BindableFloat`) or ``{"value", "source_id"}``.
* ``"settings.pattern_template_id"`` — arbitrarily deep object access.
* ``"layers[].source_id"`` — ``layers`` is a list; read ``source_id``
from every element.
* ``"calibration.lines[].picture_source_id"`` — object → list → field.
"""
from __future__ import annotations
import logging
from dataclasses import asdict, dataclass, is_dataclass
from typing import Any
logger = logging.getLogger(__name__)
@dataclass(frozen=True)
class ConnectionField:
"""One connectable reference: ``target_kind.field`` points at ``source_kind``."""
target_kind: str
"""Entity kind that *holds* the reference (the consumer / referrer)."""
field: str
"""Dot-path to the reference value (see module docstring grammar)."""
source_kind: str
"""Entity kind being referenced (the producer / source)."""
edge_type: str
"""Edge category, used by the client for colour and port grouping."""
bindable: bool = False
"""True when the slot is a :class:`BindableFloat`/``BindableColor`` value binding."""
nested: bool = False
"""True when the field lives inside a nested object/list (dotted path)."""
@property
def is_list(self) -> bool:
"""True when any path segment iterates a list (``foo[]``)."""
return "[]" in self.field
# ── Entity kinds & their human "type" attribute ────────────────────────────
# Mirrors the frontend buildGraph(): kind → the serialized field that carries
# the entity's subtype (used only for the node label / icon).
NODE_TYPE_FIELD: dict[str, str] = {
"device": "device_type",
"capture_template": "engine_type",
"pp_template": "",
"audio_template": "engine_type",
"pattern_template": "",
"picture_source": "stream_type",
"audio_source": "source_type",
"value_source": "source_type",
"color_strip_source": "source_type",
"sync_clock": "",
"output_target": "target_type",
"scene_preset": "",
"automation": "",
"cspt": "",
}
ENTITY_KINDS: tuple[str, ...] = tuple(NODE_TYPE_FIELD.keys())
# ── The registry ───────────────────────────────────────────────────────────
# NOTE: ``gradient`` and ``ha_source`` reference fields are intentionally
# omitted — they are not first-class graph node kinds, so wiring them would
# only ever produce dangling-reference noise.
CONNECTION_SCHEMA: tuple[ConnectionField, ...] = (
# ── Picture sources ──
ConnectionField("picture_source", "capture_template_id", "capture_template", "template"),
ConnectionField("picture_source", "source_stream_id", "picture_source", "picture"),
ConnectionField("picture_source", "postprocessing_template_id", "pp_template", "template"),
# ── Audio sources ──
ConnectionField("audio_source", "audio_template_id", "audio_template", "audio"),
ConnectionField("audio_source", "audio_source_id", "audio_source", "audio"),
# ── Value sources ──
ConnectionField("value_source", "audio_source_id", "audio_source", "audio"),
ConnectionField("value_source", "picture_source_id", "picture_source", "picture"),
ConnectionField("value_source", "value_source_id", "value_source", "value"),
ConnectionField("value_source", "color_strip_source_id", "color_strip_source", "colorstrip"),
# ── Color strip sources (top-level) ──
ConnectionField("color_strip_source", "picture_source_id", "picture_source", "picture"),
ConnectionField("color_strip_source", "audio_source_id", "audio_source", "audio"),
ConnectionField("color_strip_source", "clock_id", "sync_clock", "clock"),
ConnectionField("color_strip_source", "input_source_id", "color_strip_source", "colorstrip"),
ConnectionField("color_strip_source", "processing_template_id", "cspt", "template"),
# ── Color strip sources (BindableFloat value bindings) ──
*(
ConnectionField(
"color_strip_source",
f"{prop}.source_id",
"value_source",
"value",
bindable=True,
nested=True,
)
for prop in (
"smoothing",
"sensitivity",
"intensity",
"scale",
"speed",
"wind_strength",
"temperature_influence",
"sound_volume",
"timeout",
"brightness",
)
),
# ── Color strip sources (BindableColor value bindings) ──
*(
ConnectionField(
"color_strip_source",
f"{prop}.source_id",
"value_source",
"value",
bindable=True,
nested=True,
)
for prop in ("color", "color_peak", "fallback_color", "default_color")
),
# ── Color strip sources (composite layers / mapped zones / calibration) ──
ConnectionField(
"color_strip_source", "layers[].source_id", "color_strip_source", "colorstrip", nested=True
),
ConnectionField(
"color_strip_source",
"layers[].brightness_source_id",
"value_source",
"value",
bindable=True,
nested=True,
),
ConnectionField(
"color_strip_source", "layers[].processing_template_id", "cspt", "template", nested=True
),
ConnectionField(
"color_strip_source", "zones[].source_id", "color_strip_source", "colorstrip", nested=True
),
ConnectionField(
"color_strip_source",
"calibration.lines[].picture_source_id",
"picture_source",
"picture",
nested=True,
),
# ── Output targets ──
ConnectionField("output_target", "device_id", "device", "device"),
ConnectionField("output_target", "color_strip_source_id", "color_strip_source", "colorstrip"),
ConnectionField("output_target", "picture_source_id", "picture_source", "picture"),
ConnectionField(
"output_target", "brightness.source_id", "value_source", "value", bindable=True, nested=True
),
ConnectionField(
"output_target", "transition.source_id", "value_source", "value", bindable=True, nested=True
),
ConnectionField(
"output_target", "settings.pattern_template_id", "pattern_template", "template", nested=True
),
ConnectionField(
"output_target",
"settings.brightness.source_id",
"value_source",
"value",
bindable=True,
nested=True,
),
# ── Scene presets ──
ConnectionField("scene_preset", "targets[].target_id", "output_target", "scene", nested=True),
# ── Automations ──
ConnectionField("automation", "scene_preset_id", "scene_preset", "scene"),
ConnectionField("automation", "deactivation_scene_preset_id", "scene_preset", "scene"),
# ── Devices ──
ConnectionField("device", "default_css_processing_template_id", "cspt", "template"),
)
def schema_for_kind(kind: str) -> list[ConnectionField]:
"""Every connectable field whose *referrer* is ``kind``."""
return [c for c in CONNECTION_SCHEMA if c.target_kind == kind]
def schema_as_dicts() -> list[dict[str, Any]]:
"""Serialize the registry for the ``/graph/schema`` endpoint."""
return [
{
"target_kind": c.target_kind,
"field": c.field,
"source_kind": c.source_kind,
"edge_type": c.edge_type,
"bindable": c.bindable,
"nested": c.nested,
"is_list": c.is_list,
}
for c in CONNECTION_SCHEMA
]
# ── Reference extraction ────────────────────────────────────────────────────
def extract_refs(entity: dict[str, Any], field_path: str) -> list[str]:
"""Resolve a (possibly nested/list) ``field_path`` to its referenced ids.
Returns only non-empty string ids. Tolerant of missing keys, ``None``
values and unbound bindables (a plain number where an object was expected).
"""
current: list[Any] = [entity]
for segment in field_path.split("."):
is_list = segment.endswith("[]")
key = segment[:-2] if is_list else segment
nxt: list[Any] = []
for obj in current:
if not isinstance(obj, dict):
continue
val = obj.get(key)
if is_list:
if isinstance(val, list):
nxt.extend(val)
elif val is not None:
nxt.append(val)
current = nxt
return [v for v in current if isinstance(v, str) and v]
def serialize_entity(model: Any) -> dict[str, Any]:
"""Best-effort serialize a storage model to a plain dict for graph use.
Prefers ``dataclasses.asdict`` (pure structural, recurses bindables/lists,
invokes no managers), falling back to ``to_dict()`` then ``{}``.
"""
if is_dataclass(model) and not isinstance(model, type):
try:
return asdict(model)
except Exception as exc: # noqa: BLE001 — defensive: never let one model break the graph
logger.debug("graph: asdict failed for %r: %s", type(model).__name__, exc)
to_dict = getattr(model, "to_dict", None)
if callable(to_dict):
try:
result = to_dict()
if isinstance(result, dict):
return result
except Exception as exc: # noqa: BLE001
logger.debug("graph: to_dict failed for %r: %s", type(model).__name__, exc)
logger.warning(
"graph: could not serialize model %r; excluding from graph", type(model).__name__
)
return {}
# ── Topology / validation ───────────────────────────────────────────────────
def _node_from(kind: str, entity: dict[str, Any]) -> dict[str, Any] | None:
eid = entity.get("id")
if not isinstance(eid, str) or not eid:
return None
type_field = NODE_TYPE_FIELD.get(kind, "")
subtype = entity.get(type_field, "") if type_field else ""
return {
"id": eid,
"kind": kind,
"name": entity.get("name") or eid,
"type": subtype if isinstance(subtype, str) else "",
}
def build_topology(entities_by_kind: dict[str, list[dict[str, Any]]]) -> dict[str, Any]:
"""Build the full wiring graph + a validation report.
Args:
entities_by_kind: ``{kind: [serialized_entity_dict, ...]}``.
Returns a dict with ``nodes``, ``edges`` and ``issues`` (``orphans``,
``broken_refs``, ``cycles``).
"""
nodes: list[dict[str, Any]] = []
node_ids: set[str] = set()
for kind in ENTITY_KINDS:
for entity in entities_by_kind.get(kind, []):
node = _node_from(kind, entity)
if node and node["id"] not in node_ids:
node_ids.add(node["id"])
nodes.append(node)
edges: list[dict[str, Any]] = []
broken_refs: list[dict[str, str]] = []
for cf in CONNECTION_SCHEMA:
for entity in entities_by_kind.get(cf.target_kind, []):
referrer = entity.get("id")
if not isinstance(referrer, str) or not referrer:
continue
for ref in extract_refs(entity, cf.field):
if ref not in node_ids:
broken_refs.append({"ref": ref, "by": referrer, "field": cf.field})
continue
edges.append(
{
"from": ref,
"to": referrer,
"field": cf.field,
"edge_type": cf.edge_type,
"nested": cf.nested,
}
)
connected: set[str] = set()
for e in edges:
connected.add(e["from"])
connected.add(e["to"])
orphans = sorted(nid for nid in node_ids if nid not in connected)
cycles = sorted(detect_cycles(edges))
return {
"nodes": nodes,
"edges": edges,
"issues": {
"orphans": orphans,
"broken_refs": broken_refs,
"cycles": cycles,
},
}
def find_dependents(
entities_by_kind: dict[str, list[dict[str, Any]]], kind: str, entity_id: str
) -> list[dict[str, str]]:
"""Return every entity that references ``(kind, entity_id)``.
``kind`` is the kind of the *referenced* entity; matching schema entries are
those whose ``source_kind == kind``.
"""
name_by_id: dict[str, str] = {}
for k in ENTITY_KINDS:
for entity in entities_by_kind.get(k, []):
eid = entity.get("id")
if isinstance(eid, str):
name_by_id[eid] = entity.get("name") or eid
dependents: list[dict[str, str]] = []
seen: set[tuple[str, str]] = set()
for cf in CONNECTION_SCHEMA:
if cf.source_kind != kind:
continue
for entity in entities_by_kind.get(cf.target_kind, []):
referrer = entity.get("id")
if not isinstance(referrer, str):
continue
if entity_id in extract_refs(entity, cf.field):
key = (referrer, cf.field)
if key in seen:
continue
seen.add(key)
dependents.append(
{
"id": referrer,
"kind": cf.target_kind,
"name": name_by_id.get(referrer, referrer),
"field": cf.field,
}
)
return dependents
def detect_cycles(edges: list[dict[str, Any]]) -> set[str]:
"""Return every node id that participates in a directed cycle (from→to)."""
adj: dict[str, list[str]] = {}
for e in edges:
adj.setdefault(e["from"], []).append(e["to"])
WHITE, GRAY, BLACK = 0, 1, 2
color: dict[str, int] = {}
in_cycle: set[str] = set()
for start in list(adj.keys()):
if color.get(start, WHITE) != WHITE:
continue
stack: list[tuple[str, int]] = [(start, 0)]
path: list[str] = [start]
color[start] = GRAY
while stack:
node, idx = stack[-1]
neighbors = adj.get(node, [])
if idx < len(neighbors):
stack[-1] = (node, idx + 1)
nxt = neighbors[idx]
c = color.get(nxt, WHITE)
if c == GRAY:
if nxt in path:
i = path.index(nxt)
in_cycle.update(path[i:])
elif c == WHITE:
color[nxt] = GRAY
path.append(nxt)
stack.append((nxt, 0))
else:
color[node] = BLACK
if path and path[-1] == node:
path.pop()
stack.pop()
return in_cycle
def _reachable(edges: list[dict[str, Any]], start: str, goal: str) -> bool:
"""True if ``goal`` is reachable from ``start`` following from→to edges."""
if start == goal:
return True
adj: dict[str, list[str]] = {}
for e in edges:
adj.setdefault(e["from"], []).append(e["to"])
seen = {start}
queue = [start]
while queue:
cur = queue.pop()
for nxt in adj.get(cur, []):
if nxt == goal:
return True
if nxt not in seen:
seen.add(nxt)
queue.append(nxt)
return False
def would_create_cycle(edges: list[dict[str, Any]], source_id: str, target_id: str) -> bool:
"""Would wiring ``source_id`` into ``target_id`` (edge source→target) loop?
A cycle forms if ``source_id`` is already reachable from ``target_id`` via
the existing data-flow edges (so the new edge would close the loop), or the
two are the same node.
"""
if source_id == target_id:
return True
return _reachable(edges, target_id, source_id)
def _entity_exists(
entities_by_kind: dict[str, list[dict[str, Any]]], kind: str, entity_id: str
) -> bool:
return any(e.get("id") == entity_id for e in entities_by_kind.get(kind, []))
def validate_connection(
entities_by_kind: dict[str, list[dict[str, Any]]],
target_kind: str,
target_id: str,
field: str,
source_id: str,
) -> tuple[bool, str | None]:
"""Validate a proposed wiring edit before it is persisted.
Checks, in order: the field is a known connectable reference; the target
exists; (when not detaching) the source exists and is of the registry's
expected kind; and the edit would not create a dependency cycle. Returns
``(ok, error_message)``. Detaching (empty ``source_id``) is always allowed.
"""
cf = next(
(c for c in CONNECTION_SCHEMA if c.target_kind == target_kind and c.field == field),
None,
)
if cf is None:
return False, f"Unknown connection field: {target_kind}.{field}"
if cf.is_list:
# List slots (layers/zones/scene targets) hold many edges sharing the
# same (to, field); without an element index this endpoint can't model
# which one is being replaced for the cycle check. Edit those via the
# entity editor.
return False, f"List connection '{field}' must be edited via the entity editor"
if not _entity_exists(entities_by_kind, target_kind, target_id):
return False, f"Target entity not found: {target_id}"
if not source_id:
return True, None # detaching a slot is always valid
if not _entity_exists(entities_by_kind, cf.source_kind, source_id):
return False, f"Source {cf.source_kind} not found: {source_id}"
# Cycle check: ignore the edge currently occupying this slot, since the
# write replaces it.
topo = build_topology(entities_by_kind)
edges = [e for e in topo["edges"] if not (e["to"] == target_id and e["field"] == field)]
if would_create_cycle(edges, source_id, target_id):
return False, "Connection would create a dependency cycle"
return True, None