diff --git a/TODO.md b/TODO.md index ed4b06c..7661f71 100644 --- a/TODO.md +++ b/TODO.md @@ -1,47 +1,42 @@ -# Weather Source Implementation +# SQLite Migration -## Phase 1: Backend — Entity & Provider +## Phase 1: Infrastructure +- [x] Create `storage/database.py` — SQLite connection wrapper (WAL mode, thread-safe) +- [x] Create `storage/base_sqlite_store.py` — same public API as BaseJsonStore, backed by SQLite +- [x] Create `storage/migration.py` — auto-migrate JSON files to SQLite on first run +- [x] Add `database_file` to `StorageConfig` in config.py +- [x] Update demo mode path rewriting for database_file -- [x] `storage/weather_source.py` — WeatherSource dataclass -- [x] `storage/weather_source_store.py` — BaseJsonStore, CRUD, ID prefix `ws_` -- [x] `api/schemas/weather_sources.py` — Create/Update/Response Pydantic models -- [x] `api/routes/weather_sources.py` — REST CRUD + `POST /{id}/test` endpoint -- [x] `core/weather/weather_provider.py` — WeatherData, WeatherProvider ABC, OpenMeteoProvider, WMO_CONDITION_NAMES -- [x] `core/weather/weather_manager.py` — Ref-counted runtime pool, polls API, caches WeatherData -- [x] `config.py` — Add `weather_sources_file` to StorageConfig -- [x] `main.py` — Init store + manager, inject dependencies, shutdown save -- [x] `api/__init__.py` — Register router -- [x] `api/routes/backup.py` — Add to STORE_MAP +## Phase 2: Convert stores (one-by-one) +- [x] SyncClockStore +- [x] GradientStore +- [x] WeatherSourceStore +- [x] AutomationStore +- [x] ScenePresetStore +- [x] TemplateStore +- [x] PostprocessingTemplateStore +- [x] PatternTemplateStore +- [x] AudioTemplateStore +- [x] ColorStripProcessingTemplateStore +- [x] PictureSourceStore +- [x] AudioSourceStore +- [x] ValueSourceStore +- [x] DeviceStore +- [x] OutputTargetStore +- [x] ColorStripStore -## Phase 2: Backend — CSS Stream +## Phase 3: Update backup/restore +- [x] Refactor backup.py to read from SQLite (export/import/backup/restore) +- [x] Keep JSON backup format identical for compatibility +- [x] Update AutoBackupEngine to read from SQLite +- [x] Add Database to dependency injection -- [x] `core/processing/weather_stream.py` — WeatherColorStripStream with WMO palette mapping + temperature shift + thunderstorm flash -- [x] `core/processing/color_strip_stream_manager.py` — Register `"weather"` stream type + weather_manager dependency -- [x] `storage/color_strip_source.py` — WeatherColorStripSource dataclass + registry -- [x] `api/schemas/color_strip_sources.py` — Add `"weather"` to Literal + weather_source_id, temperature_influence fields -- [x] `core/processing/processor_manager.py` — Pass weather_manager through ProcessorDependencies - -## Phase 3: Frontend — Weather Source Entity - -- [x] `templates/modals/weather-source-editor.html` — Modal with provider select, lat/lon + "Use my location", update interval, test button -- [x] `static/js/features/weather-sources.ts` — Modal, CRUD, test (shows weather toast), clone, geolocation, CardSection delegation -- [x] `static/js/core/state.ts` — weatherSourcesCache + _cachedWeatherSources -- [x] `static/js/types.ts` — WeatherSource interface + ColorStripSource weather fields -- [x] `static/js/features/streams.ts` — Weather Sources CardSection + card renderer + tree nav -- [x] `templates/index.html` — Include modal template -- [x] `static/css/modal.css` — Weather location row styles - -## Phase 4: Frontend — CSS Editor Integration - -- [x] `static/js/features/color-strips.ts` — `"weather"` type, section map, handler, card renderer, populate dropdown -- [x] `static/js/core/icons.ts` — Weather icon in CSS type icons -- [x] `templates/modals/css-editor.html` — Weather section (EntitySelect for weather source, speed, temperature_influence) - -## Phase 5: i18n + Build - -- [x] `static/locales/en.json` — Weather source + CSS editor keys -- [x] `static/locales/ru.json` — Russian translations -- [x] `static/locales/zh.json` — Chinese translations -- [x] Lint: `ruff check` — passed -- [x] Build: `tsc --noEmit` + `npm run build` — passed -- [ ] Restart server + test +## Phase 4: Cleanup +- [ ] Remove individual `*_file` fields from StorageConfig (keep `database_file` only) +- [ ] Remove `atomic_write_json` usage from stores (still used by auto_backup settings) +- [ ] Remove `freeze_saves` from base_store (only `freeze_writes` needed) +- [ ] Remove BaseJsonStore (keep EntityNotFoundError — move to shared location) +- [ ] Update _save_all_stores to use _save_all() instead of _save(force=True) +- [ ] Update CLAUDE.md and server/CLAUDE.md documentation +- [ ] Remove `_json_key`/`_legacy_json_keys` references from old code +- [ ] Clean up test files to use Database fixture instead of file paths diff --git a/server/config/default_config.yaml b/server/config/default_config.yaml index d337e5d..6fe393f 100644 --- a/server/config/default_config.yaml +++ b/server/config/default_config.yaml @@ -15,12 +15,7 @@ auth: dev: "development-key-change-in-production" storage: - devices_file: "data/devices.json" - templates_file: "data/capture_templates.json" - postprocessing_templates_file: "data/postprocessing_templates.json" - picture_sources_file: "data/picture_sources.json" - output_targets_file: "data/output_targets.json" - pattern_templates_file: "data/pattern_templates.json" + database_file: "data/ledgrab.db" mqtt: enabled: false diff --git a/server/config/demo_config.yaml b/server/config/demo_config.yaml index e05343a..9b1a825 100644 --- a/server/config/demo_config.yaml +++ b/server/config/demo_config.yaml @@ -19,12 +19,7 @@ auth: demo: "demo" storage: - devices_file: "data/devices.json" - templates_file: "data/capture_templates.json" - postprocessing_templates_file: "data/postprocessing_templates.json" - picture_sources_file: "data/picture_sources.json" - output_targets_file: "data/output_targets.json" - pattern_templates_file: "data/pattern_templates.json" + database_file: "data/ledgrab.db" mqtt: enabled: false diff --git a/server/config/test_config.yaml b/server/config/test_config.yaml index 34d01e4..92c3ddb 100644 --- a/server/config/test_config.yaml +++ b/server/config/test_config.yaml @@ -11,12 +11,7 @@ auth: test_client: "eb8a89cfd33ab067751fd0e38f74ddf7ac3d75ff012fbab35a616c45c12e0c8d" storage: - devices_file: "data/test_devices.json" - templates_file: "data/capture_templates.json" - postprocessing_templates_file: "data/postprocessing_templates.json" - picture_sources_file: "data/picture_sources.json" - output_targets_file: "data/output_targets.json" - pattern_templates_file: "data/pattern_templates.json" + database_file: "data/test_ledgrab.db" logging: format: "text" diff --git a/server/src/wled_controller/api/dependencies.py b/server/src/wled_controller/api/dependencies.py index d4e0233..0aa1e5b 100644 --- a/server/src/wled_controller/api/dependencies.py +++ b/server/src/wled_controller/api/dependencies.py @@ -7,6 +7,7 @@ All getter function signatures remain unchanged for FastAPI Depends() compatibil from typing import Any, Dict, TypeVar from wled_controller.core.processing.processor_manager import ProcessorManager +from wled_controller.storage.database import Database from wled_controller.storage import DeviceStore from wled_controller.storage.template_store import TemplateStore from wled_controller.storage.postprocessing_template_store import PostprocessingTemplateStore @@ -129,6 +130,10 @@ def get_weather_manager() -> WeatherManager: return _get("weather_manager", "Weather manager") +def get_database() -> Database: + return _get("database", "Database") + + # ── Event helper ──────────────────────────────────────────────────────── @@ -157,6 +162,7 @@ def init_dependencies( device_store: DeviceStore, template_store: TemplateStore, processor_manager: ProcessorManager, + database: Database | None = None, pp_template_store: PostprocessingTemplateStore | None = None, pattern_template_store: PatternTemplateStore | None = None, picture_source_store: PictureSourceStore | None = None, @@ -178,6 +184,7 @@ def init_dependencies( ): """Initialize global dependencies.""" _deps.update({ + "database": database, "device_store": device_store, "template_store": template_store, "processor_manager": processor_manager, diff --git a/server/src/wled_controller/api/routes/backup.py b/server/src/wled_controller/api/routes/backup.py index 0f689e3..7d82a40 100644 --- a/server/src/wled_controller/api/routes/backup.py +++ b/server/src/wled_controller/api/routes/backup.py @@ -1,23 +1,20 @@ -"""System routes: backup, restore, export, import, auto-backup. +"""System routes: backup, restore, auto-backup. -Extracted from system.py to keep files under 800 lines. +All backups are SQLite database snapshots (.db files). """ import asyncio import io -import json import subprocess import sys import threading -from datetime import datetime, timezone from pathlib import Path -from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile from fastapi.responses import StreamingResponse -from wled_controller import __version__ from wled_controller.api.auth import AuthRequired -from wled_controller.api.dependencies import get_auto_backup_engine +from wled_controller.api.dependencies import get_auto_backup_engine, get_database from wled_controller.api.schemas.system import ( AutoBackupSettings, AutoBackupStatusResponse, @@ -26,38 +23,13 @@ from wled_controller.api.schemas.system import ( RestoreResponse, ) from wled_controller.core.backup.auto_backup import AutoBackupEngine -from wled_controller.config import get_config -from wled_controller.storage.base_store import freeze_saves -from wled_controller.utils import atomic_write_json, get_logger +from wled_controller.storage.database import Database, freeze_writes +from wled_controller.utils import get_logger logger = get_logger(__name__) router = APIRouter() -# --------------------------------------------------------------------------- -# Configuration backup / restore -# --------------------------------------------------------------------------- - -# Mapping: logical store name -> StorageConfig attribute name -STORE_MAP = { - "devices": "devices_file", - "capture_templates": "templates_file", - "postprocessing_templates": "postprocessing_templates_file", - "picture_sources": "picture_sources_file", - "output_targets": "output_targets_file", - "pattern_templates": "pattern_templates_file", - "color_strip_sources": "color_strip_sources_file", - "audio_sources": "audio_sources_file", - "audio_templates": "audio_templates_file", - "value_sources": "value_sources_file", - "sync_clocks": "sync_clocks_file", - "color_strip_processing_templates": "color_strip_processing_templates_file", - "automations": "automations_file", - "scene_presets": "scene_presets_file", - "gradients": "gradients_file", - "weather_sources": "weather_sources_file", -} - _SERVER_DIR = Path(__file__).resolve().parents[4] @@ -82,147 +54,74 @@ def _schedule_restart() -> None: threading.Thread(target=_restart, daemon=True).start() -@router.get("/api/v1/system/export/{store_key}", tags=["System"]) -def export_store(store_key: str, _: AuthRequired): - """Download a single entity store as a JSON file.""" - if store_key not in STORE_MAP: - raise HTTPException( - status_code=404, - detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}", - ) - config = get_config() - file_path = Path(getattr(config.storage, STORE_MAP[store_key])) - if file_path.exists(): - with open(file_path, "r", encoding="utf-8") as f: - data = json.load(f) - else: - data = {} +# --------------------------------------------------------------------------- +# Backup / restore (SQLite snapshots) +# --------------------------------------------------------------------------- - export = { - "meta": { - "format": "ledgrab-partial-export", - "format_version": 1, - "store_key": store_key, - "app_version": __version__, - "created_at": datetime.now(timezone.utc).isoformat() + "Z", - }, - "store": data, - } - content = json.dumps(export, indent=2, ensure_ascii=False) + +@router.get("/api/v1/system/backup", tags=["System"]) +def backup_config(_: AuthRequired, db: Database = Depends(get_database)): + """Download a full database backup as a .db file.""" + import tempfile + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + tmp_path = Path(tmp.name) + + try: + db.backup_to(tmp_path) + content = tmp_path.read_bytes() + finally: + tmp_path.unlink(missing_ok=True) + + from datetime import datetime, timezone timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S") - filename = f"ledgrab-{store_key}-{timestamp}.json" + filename = f"ledgrab-backup-{timestamp}.db" + return StreamingResponse( - io.BytesIO(content.encode("utf-8")), - media_type="application/json", + io.BytesIO(content), + media_type="application/octet-stream", headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) -@router.post("/api/v1/system/import/{store_key}", tags=["System"]) -async def import_store( - store_key: str, +@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"]) +async def restore_config( _: AuthRequired, file: UploadFile = File(...), - merge: bool = Query(False, description="Merge into existing data instead of replacing"), + db: Database = Depends(get_database), ): - """Upload a partial export file to replace or merge one entity store. Triggers server restart.""" - if store_key not in STORE_MAP: - raise HTTPException( - status_code=404, - detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}", - ) + """Upload a .db backup file to restore all configuration. Triggers server restart.""" + raw = await file.read() + if len(raw) > 50 * 1024 * 1024: # 50 MB limit + raise HTTPException(status_code=400, detail="Backup file too large (max 50 MB)") + + if len(raw) < 100: + raise HTTPException(status_code=400, detail="File too small to be a valid SQLite database") + + # SQLite files start with "SQLite format 3\000" + if not raw[:16].startswith(b"SQLite format 3"): + raise HTTPException(status_code=400, detail="Not a valid SQLite database file") + + import tempfile + with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp: + tmp.write(raw) + tmp_path = Path(tmp.name) try: - raw = await file.read() - if len(raw) > 10 * 1024 * 1024: - raise HTTPException(status_code=400, detail="File too large (max 10 MB)") - payload = json.loads(raw) - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}") + def _restore(): + db.restore_from(tmp_path) - # Support both full-backup format and partial-export format - if "stores" in payload and isinstance(payload.get("meta"), dict): - # Full backup: extract the specific store - if payload["meta"].get("format") not in ("ledgrab-backup",): - raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file") - stores = payload.get("stores", {}) - if store_key not in stores: - raise HTTPException(status_code=400, detail=f"Backup does not contain store '{store_key}'") - incoming = stores[store_key] - elif isinstance(payload.get("meta"), dict) and payload["meta"].get("format") == "ledgrab-partial-export": - # Partial export format - if payload["meta"].get("store_key") != store_key: - raise HTTPException( - status_code=400, - detail=f"File is for store '{payload['meta']['store_key']}', not '{store_key}'", - ) - incoming = payload.get("store", {}) - else: - raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file") + await asyncio.to_thread(_restore) + finally: + tmp_path.unlink(missing_ok=True) - if not isinstance(incoming, dict): - raise HTTPException(status_code=400, detail="Store data must be a JSON object") - - config = get_config() - file_path = Path(getattr(config.storage, STORE_MAP[store_key])) - - def _write(): - if merge and file_path.exists(): - with open(file_path, "r", encoding="utf-8") as f: - existing = json.load(f) - if isinstance(existing, dict): - existing.update(incoming) - atomic_write_json(file_path, existing) - return len(existing) - atomic_write_json(file_path, incoming) - return len(incoming) - - count = await asyncio.to_thread(_write) - freeze_saves() - logger.info(f"Imported store '{store_key}' ({count} entries, merge={merge}). Scheduling restart...") + freeze_writes() + logger.info("Database restored from uploaded backup. Scheduling restart...") _schedule_restart() - return { - "status": "imported", - "store_key": store_key, - "entries": count, - "merge": merge, - "restart_scheduled": True, - "message": f"Imported {count} entries for '{store_key}'. Server restarting...", - } - -@router.get("/api/v1/system/backup", tags=["System"]) -def backup_config(_: AuthRequired): - """Download all configuration as a single JSON backup file.""" - config = get_config() - stores = {} - for store_key, config_attr in STORE_MAP.items(): - file_path = Path(getattr(config.storage, config_attr)) - if file_path.exists(): - with open(file_path, "r", encoding="utf-8") as f: - stores[store_key] = json.load(f) - else: - stores[store_key] = {} - - backup = { - "meta": { - "format": "ledgrab-backup", - "format_version": 1, - "app_version": __version__, - "created_at": datetime.now(timezone.utc).isoformat() + "Z", - "store_count": len(stores), - }, - "stores": stores, - } - - content = json.dumps(backup, indent=2, ensure_ascii=False) - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S") - filename = f"ledgrab-backup-{timestamp}.json" - - return StreamingResponse( - io.BytesIO(content.encode("utf-8")), - media_type="application/json", - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + return RestoreResponse( + status="restored", + restart_scheduled=True, + message="Database restored from backup. Server restarting...", ) @@ -237,110 +136,12 @@ def restart_server(_: AuthRequired): @router.post("/api/v1/system/shutdown", tags=["System"]) def shutdown_server(_: AuthRequired): - """Gracefully shut down the server. - - Signals uvicorn to exit, which triggers the lifespan shutdown handler - (persists all stores to disk, stops processors, etc.). - Used by the restart script to ensure data is saved before the process exits. - """ + """Gracefully shut down the server.""" from wled_controller.server_ref import request_shutdown request_shutdown() return {"status": "shutting_down"} -@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"]) -async def restore_config( - _: AuthRequired, - file: UploadFile = File(...), -): - """Upload a backup file to restore all configuration. Triggers server restart.""" - # Read and parse - try: - raw = await file.read() - if len(raw) > 10 * 1024 * 1024: # 10 MB limit - raise HTTPException(status_code=400, detail="Backup file too large (max 10 MB)") - backup = json.loads(raw) - except json.JSONDecodeError as e: - raise HTTPException(status_code=400, detail=f"Invalid JSON file: {e}") - - # Validate envelope - meta = backup.get("meta") - if not isinstance(meta, dict) or meta.get("format") != "ledgrab-backup": - raise HTTPException(status_code=400, detail="Not a valid LED Grab backup file") - - fmt_version = meta.get("format_version", 0) - if fmt_version > 1: - raise HTTPException( - status_code=400, - detail=f"Backup format version {fmt_version} is not supported by this server version", - ) - - stores = backup.get("stores") - if not isinstance(stores, dict): - raise HTTPException(status_code=400, detail="Backup file missing 'stores' section") - - known_keys = set(STORE_MAP.keys()) - present_keys = known_keys & set(stores.keys()) - if not present_keys: - raise HTTPException(status_code=400, detail="Backup contains no recognized store data") - - for key in present_keys: - if not isinstance(stores[key], dict): - raise HTTPException(status_code=400, detail=f"Store '{key}' in backup is not a valid JSON object") - - # Guard: reject backups where every store is empty (version key only, no entities). - # This prevents accidental data wipes from restoring a backup taken when the - # server had no data loaded. - total_entities = 0 - for key in present_keys: - store_data = stores[key] - for field_key, field_val in store_data.items(): - if field_key != "version" and isinstance(field_val, dict): - total_entities += len(field_val) - if total_entities == 0: - raise HTTPException( - status_code=400, - detail="Backup contains no entity data (all stores are empty). Aborting to prevent data loss.", - ) - - # Log missing stores as warnings - missing = known_keys - present_keys - if missing: - for store_key in sorted(missing): - logger.warning(f"Restore: backup is missing store '{store_key}' — existing data will be kept") - - # Write store files atomically (in thread to avoid blocking event loop) - config = get_config() - - def _write_stores(): - count = 0 - for store_key, config_attr in STORE_MAP.items(): - if store_key in stores: - file_path = Path(getattr(config.storage, config_attr)) - atomic_write_json(file_path, stores[store_key]) - count += 1 - logger.info(f"Restored store: {store_key} -> {file_path}") - return count - - written = await asyncio.to_thread(_write_stores) - - # Freeze all store saves so the old process can't overwrite restored files - # with stale in-memory data before the restart completes. - freeze_saves() - - logger.info(f"Restore complete: {written}/{len(STORE_MAP)} stores written. Scheduling restart...") - _schedule_restart() - - return RestoreResponse( - status="restored", - stores_written=written, - stores_total=len(STORE_MAP), - missing_stores=sorted(missing) if missing else [], - restart_scheduled=True, - message=f"Restored {written} stores. Server restarting...", - ) - - # --------------------------------------------------------------------------- # Auto-backup settings & saved backups # --------------------------------------------------------------------------- @@ -419,7 +220,7 @@ def download_saved_backup( content = path.read_bytes() return StreamingResponse( io.BytesIO(content), - media_type="application/json", + media_type="application/octet-stream", headers={"Content-Disposition": f'attachment; filename="{filename}"'}, ) diff --git a/server/src/wled_controller/api/routes/system.py b/server/src/wled_controller/api/routes/system.py index d82dac3..8e46786 100644 --- a/server/src/wled_controller/api/routes/system.py +++ b/server/src/wled_controller/api/routes/system.py @@ -45,8 +45,7 @@ from wled_controller.core.capture.screen_capture import get_available_displays from wled_controller.utils import get_logger from wled_controller.storage.base_store import EntityNotFoundError -# Re-export STORE_MAP and load_external_url so existing callers still work -from wled_controller.api.routes.backup import STORE_MAP # noqa: F401 +# Re-export load_external_url so existing callers still work from wled_controller.api.routes.system_settings import load_external_url # noqa: F401 logger = get_logger(__name__) diff --git a/server/src/wled_controller/api/routes/system_settings.py b/server/src/wled_controller/api/routes/system_settings.py index fc6b9cb..4e6a6b3 100644 --- a/server/src/wled_controller/api/routes/system_settings.py +++ b/server/src/wled_controller/api/routes/system_settings.py @@ -4,15 +4,14 @@ Extracted from system.py to keep files under 800 lines. """ import asyncio -import json import logging import re -from pathlib import Path -from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect +from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect from pydantic import BaseModel from wled_controller.api.auth import AuthRequired +from wled_controller.api.dependencies import get_database from wled_controller.api.schemas.system import ( ExternalUrlRequest, ExternalUrlResponse, @@ -22,6 +21,7 @@ from wled_controller.api.schemas.system import ( MQTTSettingsResponse, ) from wled_controller.config import get_config +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) @@ -33,21 +33,9 @@ router = APIRouter() # MQTT settings # --------------------------------------------------------------------------- -_MQTT_SETTINGS_FILE: Path | None = None - -def _get_mqtt_settings_path() -> Path: - global _MQTT_SETTINGS_FILE - if _MQTT_SETTINGS_FILE is None: - cfg = get_config() - # Derive the data directory from any known storage file path - data_dir = Path(cfg.storage.devices_file).parent - _MQTT_SETTINGS_FILE = data_dir / "mqtt_settings.json" - return _MQTT_SETTINGS_FILE - - -def _load_mqtt_settings() -> dict: - """Load MQTT settings: YAML config defaults overridden by JSON overrides file.""" +def _load_mqtt_settings(db: Database) -> dict: + """Load MQTT settings: YAML config defaults overridden by DB settings.""" cfg = get_config() defaults = { "enabled": cfg.mqtt.enabled, @@ -58,31 +46,20 @@ def _load_mqtt_settings() -> dict: "client_id": cfg.mqtt.client_id, "base_topic": cfg.mqtt.base_topic, } - path = _get_mqtt_settings_path() - if path.exists(): - try: - with open(path, "r", encoding="utf-8") as f: - overrides = json.load(f) - defaults.update(overrides) - except Exception as e: - logger.warning(f"Failed to load MQTT settings override file: {e}") + overrides = db.get_setting("mqtt") + if overrides: + defaults.update(overrides) return defaults -def _save_mqtt_settings(settings: dict) -> None: - """Persist MQTT settings to the JSON override file.""" - from wled_controller.utils import atomic_write_json - atomic_write_json(_get_mqtt_settings_path(), settings) - - @router.get( "/api/v1/system/mqtt/settings", response_model=MQTTSettingsResponse, tags=["System"], ) -async def get_mqtt_settings(_: AuthRequired): +async def get_mqtt_settings(_: AuthRequired, db: Database = Depends(get_database)): """Get current MQTT broker settings. Password is masked.""" - s = _load_mqtt_settings() + s = _load_mqtt_settings(db) return MQTTSettingsResponse( enabled=s["enabled"], broker_host=s["broker_host"], @@ -99,9 +76,9 @@ async def get_mqtt_settings(_: AuthRequired): response_model=MQTTSettingsResponse, tags=["System"], ) -async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest): +async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest, db: Database = Depends(get_database)): """Update MQTT broker settings. If password is empty string, the existing password is preserved.""" - current = _load_mqtt_settings() + current = _load_mqtt_settings(db) # If caller sends an empty password, keep the existing one password = body.password if body.password else current.get("password", "") @@ -115,7 +92,7 @@ async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest): "client_id": body.client_id, "base_topic": body.base_topic, } - _save_mqtt_settings(new_settings) + db.set_setting("mqtt", new_settings) logger.info("MQTT settings updated") return MQTTSettingsResponse( @@ -133,44 +110,25 @@ async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest): # External URL setting # --------------------------------------------------------------------------- -_EXTERNAL_URL_FILE: Path | None = None - - -def _get_external_url_path() -> Path: - global _EXTERNAL_URL_FILE - if _EXTERNAL_URL_FILE is None: - cfg = get_config() - data_dir = Path(cfg.storage.devices_file).parent - _EXTERNAL_URL_FILE = data_dir / "external_url.json" - return _EXTERNAL_URL_FILE - - -def load_external_url() -> str: +def load_external_url(db: Database | None = None) -> str: """Load the external URL setting. Returns empty string if not set.""" - path = _get_external_url_path() - if path.exists(): - try: - with open(path, "r", encoding="utf-8") as f: - data = json.load(f) - return data.get("external_url", "") - except Exception: - pass + if db is None: + from wled_controller.api.dependencies import get_database + db = get_database() + data = db.get_setting("external_url") + if data: + return data.get("external_url", "") return "" -def _save_external_url(url: str) -> None: - from wled_controller.utils import atomic_write_json - atomic_write_json(_get_external_url_path(), {"external_url": url}) - - @router.get( "/api/v1/system/external-url", response_model=ExternalUrlResponse, tags=["System"], ) -async def get_external_url(_: AuthRequired): +async def get_external_url(_: AuthRequired, db: Database = Depends(get_database)): """Get the configured external base URL.""" - return ExternalUrlResponse(external_url=load_external_url()) + return ExternalUrlResponse(external_url=load_external_url(db)) @router.put( @@ -178,10 +136,10 @@ async def get_external_url(_: AuthRequired): response_model=ExternalUrlResponse, tags=["System"], ) -async def update_external_url(_: AuthRequired, body: ExternalUrlRequest): +async def update_external_url(_: AuthRequired, body: ExternalUrlRequest, db: Database = Depends(get_database)): """Set the external base URL used in webhook URLs and other user-visible URLs.""" url = body.external_url.strip().rstrip("/") - _save_external_url(url) + db.set_setting("external_url", {"external_url": url}) logger.info("External URL updated: %s", url or "(cleared)") return ExternalUrlResponse(external_url=url) diff --git a/server/src/wled_controller/api/schemas/system.py b/server/src/wled_controller/api/schemas/system.py index 80cfa1f..c8260be 100644 --- a/server/src/wled_controller/api/schemas/system.py +++ b/server/src/wled_controller/api/schemas/system.py @@ -75,12 +75,9 @@ class PerformanceResponse(BaseModel): class RestoreResponse(BaseModel): - """Response after restoring configuration backup.""" + """Response after restoring database backup.""" status: str = Field(description="Status of restore operation") - stores_written: int = Field(description="Number of stores successfully written") - stores_total: int = Field(description="Total number of known stores") - missing_stores: List[str] = Field(default_factory=list, description="Store keys not found in backup") restart_scheduled: bool = Field(description="Whether server restart was scheduled") message: str = Field(description="Human-readable status message") diff --git a/server/src/wled_controller/config.py b/server/src/wled_controller/config.py index 16487fd..ac95c4e 100644 --- a/server/src/wled_controller/config.py +++ b/server/src/wled_controller/config.py @@ -27,22 +27,7 @@ class AuthConfig(BaseSettings): class StorageConfig(BaseSettings): """Storage configuration.""" - devices_file: str = "data/devices.json" - templates_file: str = "data/capture_templates.json" - postprocessing_templates_file: str = "data/postprocessing_templates.json" - picture_sources_file: str = "data/picture_sources.json" - output_targets_file: str = "data/output_targets.json" - pattern_templates_file: str = "data/pattern_templates.json" - color_strip_sources_file: str = "data/color_strip_sources.json" - audio_sources_file: str = "data/audio_sources.json" - audio_templates_file: str = "data/audio_templates.json" - value_sources_file: str = "data/value_sources.json" - automations_file: str = "data/automations.json" - scene_presets_file: str = "data/scene_presets.json" - color_strip_processing_templates_file: str = "data/color_strip_processing_templates.json" - sync_clocks_file: str = "data/sync_clocks.json" - gradients_file: str = "data/gradients.json" - weather_sources_file: str = "data/weather_sources.json" + database_file: str = "data/ledgrab.db" class MQTTConfig(BaseSettings): diff --git a/server/src/wled_controller/core/backup/auto_backup.py b/server/src/wled_controller/core/backup/auto_backup.py index 2a82912..ba60682 100644 --- a/server/src/wled_controller/core/backup/auto_backup.py +++ b/server/src/wled_controller/core/backup/auto_backup.py @@ -1,14 +1,13 @@ -"""Auto-backup engine — periodic background backups of all configuration stores.""" +"""Auto-backup engine — periodic SQLite snapshot backups.""" import asyncio -import json import os from datetime import datetime, timedelta, timezone from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import List, Optional -from wled_controller import __version__ -from wled_controller.utils import atomic_write_json, get_logger +from wled_controller.storage.database import Database +from wled_controller.utils import get_logger logger = get_logger(__name__) @@ -19,25 +18,21 @@ DEFAULT_SETTINGS = { } # Skip the immediate-on-start backup if a recent backup exists within this window. -# Prevents rapid restarts from flooding the backup directory and rotating out -# good backups. _STARTUP_BACKUP_COOLDOWN = timedelta(minutes=5) +_BACKUP_EXT = ".db" + class AutoBackupEngine: - """Creates periodic backups of all configuration stores.""" + """Creates periodic SQLite snapshot backups of the database.""" def __init__( self, - settings_path: Path, backup_dir: Path, - store_map: Dict[str, str], - storage_config: Any, + db: Database, ): - self._settings_path = Path(settings_path) self._backup_dir = Path(backup_dir) - self._store_map = store_map - self._storage_config = storage_config + self._db = db self._task: Optional[asyncio.Task] = None self._last_backup_time: Optional[datetime] = None @@ -47,17 +42,13 @@ class AutoBackupEngine: # ─── Settings persistence ────────────────────────────────── def _load_settings(self) -> dict: - if self._settings_path.exists(): - try: - with open(self._settings_path, "r", encoding="utf-8") as f: - data = json.load(f) - return {**DEFAULT_SETTINGS, **data} - except Exception as e: - logger.warning(f"Failed to load auto-backup settings: {e}") + data = self._db.get_setting("auto_backup") + if data: + return {**DEFAULT_SETTINGS, **data} return dict(DEFAULT_SETTINGS) def _save_settings(self) -> None: - atomic_write_json(self._settings_path, { + self._db.set_setting("auto_backup", { "enabled": self._settings["enabled"], "interval_hours": self._settings["interval_hours"], "max_backups": self._settings["max_backups"], @@ -90,7 +81,7 @@ class AutoBackupEngine: def _most_recent_backup_age(self) -> timedelta | None: """Return the age of the newest backup file, or None if no backups exist.""" - files = list(self._backup_dir.glob("*.json")) + files = list(self._backup_dir.glob(f"*{_BACKUP_EXT}")) if not files: return None newest = max(files, key=lambda p: p.stat().st_mtime) @@ -99,9 +90,6 @@ class AutoBackupEngine: async def _backup_loop(self) -> None: try: - # Skip immediate backup if a recent one already exists. - # Prevents rapid restarts (crashes, restores) from flooding the - # backup directory and rotating out good backups. age = self._most_recent_backup_age() if age is None or age > _STARTUP_BACKUP_COOLDOWN: await self._perform_backup() @@ -125,44 +113,22 @@ class AutoBackupEngine: # ─── Backup operations ───────────────────────────────────── async def _perform_backup(self) -> None: - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, self._perform_backup_sync) + await asyncio.to_thread(self._perform_backup_sync) def _perform_backup_sync(self) -> None: - stores = {} - for store_key, config_attr in self._store_map.items(): - file_path = Path(getattr(self._storage_config, config_attr)) - if file_path.exists(): - with open(file_path, "r", encoding="utf-8") as f: - stores[store_key] = json.load(f) - else: - stores[store_key] = {} - now = datetime.now(timezone.utc) - backup = { - "meta": { - "format": "ledgrab-backup", - "format_version": 1, - "app_version": __version__, - "created_at": now.isoformat(), - "store_count": len(stores), - "auto_backup": True, - }, - "stores": stores, - } - timestamp = now.strftime("%Y-%m-%dT%H%M%S") - filename = f"ledgrab-autobackup-{timestamp}.json" + filename = f"ledgrab-backup-{timestamp}{_BACKUP_EXT}" file_path = self._backup_dir / filename - atomic_write_json(file_path, backup) + self._db.backup_to(file_path) self._last_backup_time = now - logger.info(f"Auto-backup created: {filename}") + logger.info(f"Backup created: {filename}") def _prune_old_backups(self) -> None: max_backups = self._settings["max_backups"] - files = sorted(self._backup_dir.glob("*.json"), key=lambda p: p.stat().st_mtime) + files = sorted(self._backup_dir.glob(f"*{_BACKUP_EXT}"), key=lambda p: p.stat().st_mtime) excess = len(files) - max_backups if excess > 0: for f in files[:excess]: @@ -195,7 +161,6 @@ class AutoBackupEngine: self._settings["max_backups"] = max_backups self._save_settings() - # Restart or stop the loop if enabled: self._start_loop() logger.info( @@ -205,14 +170,12 @@ class AutoBackupEngine: self._cancel_loop() logger.info("Auto-backup disabled") - # Prune if max_backups was reduced self._prune_old_backups() - return self.get_settings() def list_backups(self) -> List[dict]: backups = [] - for f in sorted(self._backup_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True): + for f in sorted(self._backup_dir.glob(f"*{_BACKUP_EXT}"), key=lambda p: p.stat().st_mtime, reverse=True): stat = f.stat() backups.append({ "filename": f.name, @@ -226,7 +189,6 @@ class AutoBackupEngine: if not filename or os.sep in filename or "/" in filename or ".." in filename: raise ValueError("Invalid filename") target = (self._backup_dir / filename).resolve() - # Ensure resolved path is still inside the backup directory if not target.is_relative_to(self._backup_dir.resolve()): raise ValueError("Invalid filename") return target @@ -235,7 +197,6 @@ class AutoBackupEngine: """Manually trigger a backup and prune old ones. Returns the created backup info.""" await self._perform_backup() self._prune_old_backups() - # Return the most recent backup entry backups = self.list_backups() return backups[0] if backups else {} diff --git a/server/src/wled_controller/core/demo_seed.py b/server/src/wled_controller/core/demo_seed.py index 82f64ce..8298447 100644 --- a/server/src/wled_controller/core/demo_seed.py +++ b/server/src/wled_controller/core/demo_seed.py @@ -1,15 +1,14 @@ """Seed data generator for demo mode. -Populates the demo data directory with sample entities on first run, +Populates the demo SQLite database with sample entities on first run, giving new users a realistic out-of-the-box experience without needing real hardware. """ import json from datetime import datetime, timezone -from pathlib import Path -from wled_controller.config import StorageConfig +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) @@ -50,63 +49,48 @@ _SCENE_ID = "scene_demo0001" _NOW = datetime.now(timezone.utc).isoformat() -def _write_store(path: Path, json_key: str, items: dict) -> None: - """Write a store JSON file with version wrapper.""" - path.parent.mkdir(parents=True, exist_ok=True) - data = { - "version": "1.0.0", - json_key: items, - } - path.write_text(json.dumps(data, indent=2), encoding="utf-8") - logger.info(f"Seeded {len(items)} {json_key} -> {path}") +def _insert_entities(db: Database, table: str, items: dict) -> None: + """Insert entity dicts into a SQLite table.""" + rows = [] + for entity_id, entity_data in items.items(): + name = entity_data.get("name", "") + data_json = json.dumps(entity_data, ensure_ascii=False) + rows.append((entity_id, name, data_json)) + if rows: + db.bulk_insert(table, rows) + logger.info(f"Seeded {len(rows)} entities into {table}") -def _has_data(storage_config: StorageConfig) -> bool: - """Check if any demo store file already has entities.""" - for field_name in storage_config.model_fields: - value = getattr(storage_config, field_name) - if not isinstance(value, str): - continue - p = Path(value) - if p.exists() and p.stat().st_size > 20: - # File exists and is non-trivial — check if it has entities - try: - raw = json.loads(p.read_text(encoding="utf-8")) - for key, val in raw.items(): - if key != "version" and isinstance(val, dict) and val: - return True - except Exception: - pass - return False +def seed_demo_data(db: Database) -> None: + """Populate demo database with sample entities. - -def seed_demo_data(storage_config: StorageConfig) -> None: - """Populate demo data directory with sample entities. - - Only runs when the demo data directory is empty (no existing entities). + Only runs when the database has no entities in any table. Must be called BEFORE store constructors run so they load the seeded data. """ - if _has_data(storage_config): - logger.info("Demo data already exists — skipping seed") - return + # Check if any table already has data + for table in ["devices", "output_targets", "color_strip_sources", + "picture_sources", "audio_sources", "scene_presets"]: + if db.table_exists_with_data(table): + logger.info("Demo data already exists — skipping seed") + return logger.info("Seeding demo data for first-run experience") - _seed_devices(Path(storage_config.devices_file)) - _seed_capture_templates(Path(storage_config.templates_file)) - _seed_output_targets(Path(storage_config.output_targets_file)) - _seed_picture_sources(Path(storage_config.picture_sources_file)) - _seed_color_strip_sources(Path(storage_config.color_strip_sources_file)) - _seed_audio_sources(Path(storage_config.audio_sources_file)) - _seed_scene_presets(Path(storage_config.scene_presets_file)) + _insert_entities(db, "devices", _build_devices()) + _insert_entities(db, "capture_templates", _build_capture_templates()) + _insert_entities(db, "output_targets", _build_output_targets()) + _insert_entities(db, "picture_sources", _build_picture_sources()) + _insert_entities(db, "color_strip_sources", _build_color_strip_sources()) + _insert_entities(db, "audio_sources", _build_audio_sources()) + _insert_entities(db, "scene_presets", _build_scene_presets()) logger.info("Demo seed data complete") # ── Devices ──────────────────────────────────────────────────────── -def _seed_devices(path: Path) -> None: - devices = { +def _build_devices() -> dict: + return { _DEVICE_IDS["strip"]: { "id": _DEVICE_IDS["strip"], "name": "Demo LED Strip", @@ -138,13 +122,12 @@ def _seed_devices(path: Path) -> None: "updated_at": _NOW, }, } - _write_store(path, "devices", devices) # ── Capture Templates ────────────────────────────────────────────── -def _seed_capture_templates(path: Path) -> None: - templates = { +def _build_capture_templates() -> dict: + return { _TPL_ID: { "id": _TPL_ID, "name": "Demo Capture", @@ -156,13 +139,12 @@ def _seed_capture_templates(path: Path) -> None: "updated_at": _NOW, }, } - _write_store(path, "templates", templates) # ── Output Targets ───────────────────────────────────────────────── -def _seed_output_targets(path: Path) -> None: - targets = { +def _build_output_targets() -> dict: + return { _TARGET_IDS["strip"]: { "id": _TARGET_IDS["strip"], "name": "Strip — Gradient", @@ -200,13 +182,12 @@ def _seed_output_targets(path: Path) -> None: "updated_at": _NOW, }, } - _write_store(path, "output_targets", targets) # ── Picture Sources ──────────────────────────────────────────────── -def _seed_picture_sources(path: Path) -> None: - sources = { +def _build_picture_sources() -> dict: + return { _PS_IDS["main"]: { "id": _PS_IDS["main"], "name": "Demo Display 1080p", @@ -218,7 +199,6 @@ def _seed_picture_sources(path: Path) -> None: "tags": ["demo"], "created_at": _NOW, "updated_at": _NOW, - # Nulls for non-applicable subclass fields "source_stream_id": None, "postprocessing_template_id": None, "image_source": None, @@ -253,13 +233,12 @@ def _seed_picture_sources(path: Path) -> None: "clock_id": None, }, } - _write_store(path, "picture_sources", sources) # ── Color Strip Sources ──────────────────────────────────────────── -def _seed_color_strip_sources(path: Path) -> None: - sources = { +def _build_color_strip_sources() -> dict: + return { _CSS_IDS["gradient"]: { "id": _CSS_IDS["gradient"], "name": "Rainbow Gradient", @@ -338,13 +317,12 @@ def _seed_color_strip_sources(path: Path) -> None: "updated_at": _NOW, }, } - _write_store(path, "color_strip_sources", sources) # ── Audio Sources ────────────────────────────────────────────────── -def _seed_audio_sources(path: Path) -> None: - sources = { +def _build_audio_sources() -> dict: + return { _AS_IDS["system"]: { "id": _AS_IDS["system"], "name": "Demo System Audio", @@ -356,7 +334,6 @@ def _seed_audio_sources(path: Path) -> None: "tags": ["demo"], "created_at": _NOW, "updated_at": _NOW, - # Forward-compat null fields "audio_source_id": None, "channel": None, }, @@ -370,19 +347,17 @@ def _seed_audio_sources(path: Path) -> None: "tags": ["demo"], "created_at": _NOW, "updated_at": _NOW, - # Forward-compat null fields "device_index": None, "is_loopback": None, "audio_template_id": None, }, } - _write_store(path, "audio_sources", sources) # ── Scene Presets ────────────────────────────────────────────────── -def _seed_scene_presets(path: Path) -> None: - presets = { +def _build_scene_presets() -> dict: + return { _SCENE_ID: { "id": _SCENE_ID, "name": "Demo Ambient", @@ -409,4 +384,3 @@ def _seed_scene_presets(path: Path) -> None: "updated_at": _NOW, }, } - _write_store(path, "scene_presets", presets) diff --git a/server/src/wled_controller/main.py b/server/src/wled_controller/main.py index 5210c4a..ea59d94 100644 --- a/server/src/wled_controller/main.py +++ b/server/src/wled_controller/main.py @@ -41,7 +41,7 @@ from wled_controller.core.mqtt.mqtt_service import MQTTService from wled_controller.core.devices.mqtt_client import set_mqtt_service from wled_controller.core.backup.auto_backup import AutoBackupEngine from wled_controller.core.processing.os_notification_listener import OsNotificationListener -from wled_controller.api.routes.system import STORE_MAP +from wled_controller.storage.database import Database from wled_controller.utils import setup_logging, get_logger, install_broadcast_handler # Initialize logging @@ -52,29 +52,32 @@ logger = get_logger(__name__) # Get configuration config = get_config() -# Seed demo data before stores are loaded (first-run only) +# Initialize SQLite database +db = Database(config.storage.database_file) + +# Seed demo data after DB is ready (first-run only) if config.demo: from wled_controller.core.demo_seed import seed_demo_data - seed_demo_data(config.storage) + seed_demo_data(db) # Initialize storage and processing -device_store = DeviceStore(config.storage.devices_file) -template_store = TemplateStore(config.storage.templates_file) -pp_template_store = PostprocessingTemplateStore(config.storage.postprocessing_templates_file) -picture_source_store = PictureSourceStore(config.storage.picture_sources_file) -output_target_store = OutputTargetStore(config.storage.output_targets_file) -pattern_template_store = PatternTemplateStore(config.storage.pattern_templates_file) -color_strip_store = ColorStripStore(config.storage.color_strip_sources_file) -audio_source_store = AudioSourceStore(config.storage.audio_sources_file) -audio_template_store = AudioTemplateStore(config.storage.audio_templates_file) -value_source_store = ValueSourceStore(config.storage.value_sources_file) -automation_store = AutomationStore(config.storage.automations_file) -scene_preset_store = ScenePresetStore(config.storage.scene_presets_file) -sync_clock_store = SyncClockStore(config.storage.sync_clocks_file) -cspt_store = ColorStripProcessingTemplateStore(config.storage.color_strip_processing_templates_file) -gradient_store = GradientStore(config.storage.gradients_file) +device_store = DeviceStore(db) +template_store = TemplateStore(db) +pp_template_store = PostprocessingTemplateStore(db) +picture_source_store = PictureSourceStore(db) +output_target_store = OutputTargetStore(db) +pattern_template_store = PatternTemplateStore(db) +color_strip_store = ColorStripStore(db) +audio_source_store = AudioSourceStore(db) +audio_template_store = AudioTemplateStore(db) +value_source_store = ValueSourceStore(db) +automation_store = AutomationStore(db) +scene_preset_store = ScenePresetStore(db) +sync_clock_store = SyncClockStore(db) +cspt_store = ColorStripProcessingTemplateStore(db) +gradient_store = GradientStore(db) gradient_store.migrate_palette_references(color_strip_store) -weather_source_store = WeatherSourceStore(config.storage.weather_sources_file) +weather_source_store = WeatherSourceStore(db) sync_clock_manager = SyncClockManager(sync_clock_store) weather_manager = WeatherManager(weather_source_store) @@ -156,34 +159,18 @@ async def lifespan(app: FastAPI): device_store=device_store, ) - # Create auto-backup engine — derive paths from storage config so that + # Create auto-backup engine — derive paths from database location so that # demo mode auto-backups go to data/demo/ instead of data/. - _data_dir = Path(config.storage.devices_file).parent + _data_dir = Path(config.storage.database_file).parent auto_backup_engine = AutoBackupEngine( - settings_path=_data_dir / "auto_backup_settings.json", backup_dir=_data_dir / "backups", - store_map=STORE_MAP, - storage_config=config.storage, + db=db, ) - # Verify STORE_MAP covers all StorageConfig file fields. - # Catches missed additions early (at startup) rather than silently - # excluding new stores from backups. - storage_attrs = { - attr for attr in config.storage.model_fields - if attr.endswith("_file") - } - mapped_attrs = set(STORE_MAP.values()) - unmapped = storage_attrs - mapped_attrs - if unmapped: - logger.warning( - f"StorageConfig fields not in STORE_MAP (missing from backups): " - f"{sorted(unmapped)}" - ) - # Initialize API dependencies init_dependencies( device_store, template_store, processor_manager, + database=db, pp_template_store=pp_template_store, pattern_template_store=pattern_template_store, picture_source_store=picture_source_store, diff --git a/server/src/wled_controller/static/js/app.ts b/server/src/wled_controller/static/js/app.ts index dc474ae..b339161 100644 --- a/server/src/wled_controller/static/js/app.ts +++ b/server/src/wled_controller/static/js/app.ts @@ -191,10 +191,9 @@ import { import { openSettingsModal, closeSettingsModal, switchSettingsTab, downloadBackup, handleRestoreFileSelected, - saveAutoBackupSettings, restoreSavedBackup, downloadSavedBackup, deleteSavedBackup, + saveAutoBackupSettings, triggerBackupNow, restoreSavedBackup, downloadSavedBackup, deleteSavedBackup, restartServer, saveMqttSettings, loadApiKeysList, - downloadPartialExport, handlePartialImportFileSelected, connectLogViewer, disconnectLogViewer, clearLogViewer, applyLogFilter, openLogOverlay, closeLogOverlay, loadLogLevel, setLogLevel, @@ -536,21 +535,20 @@ Object.assign(window, { openCommandPalette, closeCommandPalette, - // settings (tabs / backup / restore / auto-backup / MQTT / partial export-import / api keys / log level) + // settings (tabs / backup / restore / auto-backup / MQTT / api keys / log level) openSettingsModal, closeSettingsModal, switchSettingsTab, downloadBackup, handleRestoreFileSelected, saveAutoBackupSettings, + triggerBackupNow, restoreSavedBackup, downloadSavedBackup, deleteSavedBackup, restartServer, saveMqttSettings, loadApiKeysList, - downloadPartialExport, - handlePartialImportFileSelected, connectLogViewer, disconnectLogViewer, clearLogViewer, diff --git a/server/src/wled_controller/static/js/features/settings.ts b/server/src/wled_controller/static/js/features/settings.ts index 5d03793..261b31a 100644 --- a/server/src/wled_controller/static/js/features/settings.ts +++ b/server/src/wled_controller/static/js/features/settings.ts @@ -419,6 +419,22 @@ export async function saveAutoBackupSettings(): Promise { } } +export async function triggerBackupNow(): Promise { + try { + const resp = await fetchWithAuth('/system/auto-backup/trigger', { method: 'POST' }); + if (!resp.ok) { + const err = await resp.json().catch(() => ({})); + throw new Error(err.detail || `HTTP ${resp.status}`); + } + showToast(t('settings.auto_backup.backup_created'), 'success'); + loadBackupList(); + loadAutoBackupSettings(); + } catch (err) { + console.error('Backup failed:', err); + showToast(t('settings.auto_backup.backup_error') + ': ' + err.message, 'error'); + } +} + // ─── Saved backup list ──────────────────────────────────── export async function loadBackupList(): Promise { @@ -566,76 +582,6 @@ export async function loadApiKeysList(): Promise { } } -// ─── Partial Export / Import ─────────────────────────────────── - -export async function downloadPartialExport(): Promise { - const storeKey = (document.getElementById('settings-partial-store') as HTMLSelectElement).value; - try { - const resp = await fetchWithAuth(`/system/export/${encodeURIComponent(storeKey)}`, { timeout: 30000 }); - if (!resp.ok) { - const err = await resp.json().catch(() => ({})); - throw new Error(err.detail || `HTTP ${resp.status}`); - } - const blob = await resp.blob(); - const disposition = resp.headers.get('Content-Disposition') || ''; - const match = disposition.match(/filename="(.+?)"/); - const filename = match ? match[1] : `ledgrab-${storeKey}.json`; - - const a = document.createElement('a'); - a.href = URL.createObjectURL(blob); - a.download = filename; - document.body.appendChild(a); - a.click(); - a.remove(); - URL.revokeObjectURL(a.href); - - showToast(t('settings.partial.export_success'), 'success'); - } catch (err) { - console.error('Partial export failed:', err); - showToast(t('settings.partial.export_error') + ': ' + err.message, 'error'); - } -} - -export async function handlePartialImportFileSelected(input: HTMLInputElement): Promise { - const file = input.files![0]; - input.value = ''; - if (!file) return; - - const storeKey = (document.getElementById('settings-partial-store') as HTMLSelectElement).value; - const merge = (document.getElementById('settings-partial-merge') as HTMLInputElement).checked; - const confirmMsg = merge - ? t('settings.partial.import_confirm_merge').replace('{store}', storeKey) - : t('settings.partial.import_confirm_replace').replace('{store}', storeKey); - - const confirmed = await showConfirm(confirmMsg); - if (!confirmed) return; - - try { - const formData = new FormData(); - formData.append('file', file); - - const url = `${API_BASE}/system/import/${encodeURIComponent(storeKey)}?merge=${merge}`; - const resp = await fetch(url, { - method: 'POST', - headers: { 'Authorization': `Bearer ${apiKey}` }, - body: formData, - }); - - if (!resp.ok) { - const err = await resp.json().catch(() => ({})); - throw new Error(err.detail || `HTTP ${resp.status}`); - } - - const data = await resp.json(); - showToast(data.message || t('settings.partial.import_success'), 'success'); - settingsModal.forceClose(); - - } catch (err) { - console.error('Partial import failed:', err); - showToast(t('settings.partial.import_error') + ': ' + err.message, 'error'); - } -} - // ─── Log Level ──────────────────────────────────────────────── export async function loadLogLevel(): Promise { diff --git a/server/src/wled_controller/static/js/global.d.ts b/server/src/wled_controller/static/js/global.d.ts index cccb4b2..481273b 100644 --- a/server/src/wled_controller/static/js/global.d.ts +++ b/server/src/wled_controller/static/js/global.d.ts @@ -358,14 +358,13 @@ interface Window { downloadBackup: (...args: any[]) => any; handleRestoreFileSelected: (...args: any[]) => any; saveAutoBackupSettings: (...args: any[]) => any; + triggerBackupNow: (...args: any[]) => any; restoreSavedBackup: (...args: any[]) => any; downloadSavedBackup: (...args: any[]) => any; deleteSavedBackup: (...args: any[]) => any; restartServer: (...args: any[]) => any; saveMqttSettings: (...args: any[]) => any; loadApiKeysList: (...args: any[]) => any; - downloadPartialExport: (...args: any[]) => any; - handlePartialImportFileSelected: (...args: any[]) => any; connectLogViewer: (...args: any[]) => any; disconnectLogViewer: (...args: any[]) => any; clearLogViewer: (...args: any[]) => any; diff --git a/server/src/wled_controller/static/locales/en.json b/server/src/wled_controller/static/locales/en.json index c074398..5abab89 100644 --- a/server/src/wled_controller/static/locales/en.json +++ b/server/src/wled_controller/static/locales/en.json @@ -1591,6 +1591,9 @@ "settings.auto_backup.save": "Save Settings", "settings.auto_backup.saved": "Auto-backup settings saved", "settings.auto_backup.save_error": "Failed to save auto-backup settings", + "settings.auto_backup.backup_now": "Backup Now", + "settings.auto_backup.backup_created": "Backup created", + "settings.auto_backup.backup_error": "Backup failed", "settings.auto_backup.last_backup": "Last backup", "settings.auto_backup.never": "Never", "settings.saved_backups.label": "Saved Backups", diff --git a/server/src/wled_controller/static/locales/ru.json b/server/src/wled_controller/static/locales/ru.json index c104208..722ad8b 100644 --- a/server/src/wled_controller/static/locales/ru.json +++ b/server/src/wled_controller/static/locales/ru.json @@ -1518,6 +1518,9 @@ "settings.auto_backup.save": "Сохранить настройки", "settings.auto_backup.saved": "Настройки авто-бэкапа сохранены", "settings.auto_backup.save_error": "Не удалось сохранить настройки авто-бэкапа", + "settings.auto_backup.backup_now": "Создать бэкап", + "settings.auto_backup.backup_created": "Бэкап создан", + "settings.auto_backup.backup_error": "Ошибка создания бэкапа", "settings.auto_backup.last_backup": "Последний бэкап", "settings.auto_backup.never": "Никогда", "settings.saved_backups.label": "Сохранённые копии", diff --git a/server/src/wled_controller/static/locales/zh.json b/server/src/wled_controller/static/locales/zh.json index 293a245..4033ac4 100644 --- a/server/src/wled_controller/static/locales/zh.json +++ b/server/src/wled_controller/static/locales/zh.json @@ -1518,6 +1518,9 @@ "settings.auto_backup.save": "保存设置", "settings.auto_backup.saved": "自动备份设置已保存", "settings.auto_backup.save_error": "保存自动备份设置失败", + "settings.auto_backup.backup_now": "立即备份", + "settings.auto_backup.backup_created": "备份已创建", + "settings.auto_backup.backup_error": "备份失败", "settings.auto_backup.last_backup": "上次备份", "settings.auto_backup.never": "从未", "settings.saved_backups.label": "已保存的备份", diff --git a/server/src/wled_controller/storage/audio_source_store.py b/server/src/wled_controller/storage/audio_source_store.py index 07a7ce9..b66851d 100644 --- a/server/src/wled_controller/storage/audio_source_store.py +++ b/server/src/wled_controller/storage/audio_source_store.py @@ -1,4 +1,4 @@ -"""Audio source storage using JSON files.""" +"""Audio source storage using SQLite.""" import uuid from datetime import datetime, timezone @@ -11,7 +11,8 @@ from wled_controller.storage.audio_source import ( MonoAudioSource, MultichannelAudioSource, ) -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.utils import resolve_ref from wled_controller.utils import get_logger @@ -29,18 +30,18 @@ class ResolvedAudioSource(NamedTuple): freq_high: Optional[float] = None -class AudioSourceStore(BaseJsonStore[AudioSource]): +class AudioSourceStore(BaseSqliteStore[AudioSource]): """Persistent storage for audio sources.""" - _json_key = "audio_sources" + _table_name = "audio_sources" _entity_name = "Audio source" - def __init__(self, file_path: str): - super().__init__(file_path, AudioSource.from_dict) + def __init__(self, db: Database): + super().__init__(db, AudioSource.from_dict) # Backward-compatible aliases - get_all_sources = BaseJsonStore.get_all - get_source = BaseJsonStore.get + get_all_sources = BaseSqliteStore.get_all + get_source = BaseSqliteStore.get def get_mono_sources(self) -> List[MonoAudioSource]: """Return only mono audio sources (for CSS dropdown).""" @@ -111,7 +112,7 @@ class AudioSourceStore(BaseJsonStore[AudioSource]): ) self._items[sid] = source - self._save() + self._save_item(sid, source) logger.info(f"Created audio source: {name} ({sid}, type={source_type})") return source @@ -185,7 +186,7 @@ class AudioSourceStore(BaseJsonStore[AudioSource]): source.freq_high = freq_high source.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(source_id, source) logger.info(f"Updated audio source: {source_id}") return source @@ -207,7 +208,7 @@ class AudioSourceStore(BaseJsonStore[AudioSource]): ) del self._items[source_id] - self._save() + self._delete_item(source_id) logger.info(f"Deleted audio source: {source_id}") diff --git a/server/src/wled_controller/storage/audio_template_store.py b/server/src/wled_controller/storage/audio_template_store.py index d605d39..27bf7c3 100644 --- a/server/src/wled_controller/storage/audio_template_store.py +++ b/server/src/wled_controller/storage/audio_template_store.py @@ -1,4 +1,4 @@ -"""Audio template storage using JSON files.""" +"""Audio template storage using SQLite.""" import uuid from datetime import datetime, timezone @@ -6,30 +6,31 @@ from typing import Any, Dict, List, Optional from wled_controller.core.audio.factory import AudioEngineRegistry from wled_controller.storage.audio_template import AudioCaptureTemplate -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) -class AudioTemplateStore(BaseJsonStore[AudioCaptureTemplate]): +class AudioTemplateStore(BaseSqliteStore[AudioCaptureTemplate]): """Storage for audio capture templates. - All templates are persisted to the JSON file. + All templates are persisted to the database. On startup, if no templates exist, one is auto-created using the highest-priority available engine. """ - _json_key = "templates" + _table_name = "audio_templates" _entity_name = "Audio capture template" - def __init__(self, file_path: str): - super().__init__(file_path, AudioCaptureTemplate.from_dict) + def __init__(self, db: Database): + super().__init__(db, AudioCaptureTemplate.from_dict) self._ensure_initial_template() # Backward-compatible aliases - get_all_templates = BaseJsonStore.get_all - get_template = BaseJsonStore.get + get_all_templates = BaseSqliteStore.get_all + get_template = BaseSqliteStore.get def _ensure_initial_template(self) -> None: """Auto-create a template if none exist, using the best available engine.""" @@ -93,7 +94,7 @@ class AudioTemplateStore(BaseJsonStore[AudioCaptureTemplate]): ) self._items[template_id] = template - self._save() + self._save_item(template_id, template) logger.info(f"Created audio template: {name} ({template_id})") return template @@ -121,7 +122,7 @@ class AudioTemplateStore(BaseJsonStore[AudioCaptureTemplate]): template.tags = tags template.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(template_id, template) logger.info(f"Updated audio template: {template_id}") return template @@ -152,5 +153,5 @@ class AudioTemplateStore(BaseJsonStore[AudioCaptureTemplate]): ) del self._items[template_id] - self._save() + self._delete_item(template_id) logger.info(f"Deleted audio template: {template_id}") diff --git a/server/src/wled_controller/storage/automation_store.py b/server/src/wled_controller/storage/automation_store.py index 5073f80..ec54791 100644 --- a/server/src/wled_controller/storage/automation_store.py +++ b/server/src/wled_controller/storage/automation_store.py @@ -1,27 +1,28 @@ -"""Automation storage using JSON files.""" +"""Automation storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional from wled_controller.storage.automation import Automation, Condition -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) -class AutomationStore(BaseJsonStore[Automation]): - _json_key = "automations" +class AutomationStore(BaseSqliteStore[Automation]): + _table_name = "automations" _entity_name = "Automation" - def __init__(self, file_path: str): - super().__init__(file_path, Automation.from_dict) + def __init__(self, db: Database): + super().__init__(db, Automation.from_dict) # Backward-compatible aliases - get_all_automations = BaseJsonStore.get_all - get_automation = BaseJsonStore.get - delete_automation = BaseJsonStore.delete + get_all_automations = BaseSqliteStore.get_all + get_automation = BaseSqliteStore.get + delete_automation = BaseSqliteStore.delete def create_automation( self, @@ -56,7 +57,7 @@ class AutomationStore(BaseJsonStore[Automation]): ) self._items[automation_id] = automation - self._save() + self._save_item(automation_id, automation) logger.info(f"Created automation: {name} ({automation_id})") return automation @@ -93,6 +94,6 @@ class AutomationStore(BaseJsonStore[Automation]): automation.tags = tags automation.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(automation_id, automation) logger.info(f"Updated automation: {automation_id}") return automation diff --git a/server/src/wled_controller/storage/base_sqlite_store.py b/server/src/wled_controller/storage/base_sqlite_store.py new file mode 100644 index 0000000..0901fc9 --- /dev/null +++ b/server/src/wled_controller/storage/base_sqlite_store.py @@ -0,0 +1,168 @@ +"""Base class for SQLite-backed entity stores. + +Drop-in replacement for BaseJsonStore with the same public API. +Each store keeps an in-memory cache (``_items``) for fast reads; +writes go through to SQLite immediately (write-through cache). +""" + +import asyncio +import threading +from typing import Callable, Dict, Generic, List, TypeVar + +from wled_controller.storage.database import Database +from wled_controller.utils import get_logger + +T = TypeVar("T") +logger = get_logger(__name__) + + +class BaseSqliteStore(Generic[T]): + """SQLite-backed entity store with the same API as BaseJsonStore. + + Subclasses must set class attributes: + - ``_table_name``: SQL table name (e.g. ``"sync_clocks"``) + - ``_entity_name``: human label for errors (e.g. ``"Sync clock"``) + """ + + _table_name: str + _entity_name: str + + def __init__(self, db: Database, deserializer: Callable[[dict], T]): + self._db = db + self._items: Dict[str, T] = {} + self._deserializer = deserializer + self._lock = threading.RLock() + self._load() + + # -- I/O ----------------------------------------------------------------- + + def _load(self) -> None: + """Load all rows from SQLite into the in-memory cache.""" + rows = self._db.load_all(self._table_name) + loaded = 0 + for item_dict in rows: + item_id = item_dict.get("id") + if not item_id: + logger.error(f"Skipping {self._entity_name} row with no id") + continue + try: + self._items[item_id] = self._deserializer(item_dict) + loaded += 1 + except Exception as e: + logger.error( + f"Failed to load {self._entity_name} {item_id}: {e}", + exc_info=True, + ) + + if loaded > 0: + logger.info(f"Loaded {loaded} {self._table_name} from database") + logger.info( + f"{self._entity_name} store initialized with {len(self._items)} items" + ) + + def _save_item(self, item_id: str, item: T) -> None: + """Persist a single item to SQLite (write-through).""" + data = item.to_dict() + name = data.get("name", "") + self._db.upsert(self._table_name, item_id, name, data) + + def _delete_item(self, item_id: str) -> None: + """Delete a single item from SQLite.""" + self._db.delete_row(self._table_name, item_id) + + def _save_all(self, *, force: bool = False) -> None: + """Persist all items to SQLite. + + Used during shutdown to ensure in-memory state is flushed. + When ``force`` is True, bypasses the frozen-writes check. + """ + from wled_controller.storage.database import _writes_frozen + if _writes_frozen and not force: + logger.warning(f"Save blocked (frozen after restore): {self._table_name}") + return + + items_to_write = [] + with self._lock: + for item_id, item in self._items.items(): + data = item.to_dict() + import json + items_to_write.append(( + item_id, + data.get("name", ""), + json.dumps(data, ensure_ascii=False), + )) + + if items_to_write: + # Use transaction for atomicity: clear + re-insert + with self._db.transaction() as conn: + conn.execute(f"DELETE FROM [{self._table_name}]") + conn.executemany( + f"INSERT INTO [{self._table_name}] (id, name, data) VALUES (?, ?, ?)", + items_to_write, + ) + + # -- Backward compat: _save() used by subclass create/update methods ----- + + def _save(self, *, force: bool = False) -> None: + """Compatibility shim: save all items. + + Subclasses that call ``self._save()`` after mutating ``self._items`` + will trigger a full flush. For better performance, prefer calling + ``self._save_item(id, item)`` for single-entity mutations. + """ + self._save_all(force=force) + + async def _save_async(self) -> None: + """Async wrapper — runs ``_save()`` in a thread.""" + await asyncio.to_thread(self._save) + + # -- Common CRUD (identical API to BaseJsonStore) ------------------------ + + def get_all(self) -> List[T]: + with self._lock: + return list(self._items.values()) + + def get(self, item_id: str) -> T: + with self._lock: + if item_id not in self._items: + from wled_controller.storage.base_store import EntityNotFoundError + raise EntityNotFoundError(f"{self._entity_name} not found: {item_id}") + return self._items[item_id] + + def delete(self, item_id: str) -> None: + with self._lock: + if item_id not in self._items: + from wled_controller.storage.base_store import EntityNotFoundError + raise EntityNotFoundError(f"{self._entity_name} not found: {item_id}") + del self._items[item_id] + self._delete_item(item_id) + logger.info(f"Deleted {self._entity_name}: {item_id}") + + async def async_delete(self, item_id: str) -> None: + """Async version of ``delete()``.""" + with self._lock: + if item_id not in self._items: + from wled_controller.storage.base_store import EntityNotFoundError + raise EntityNotFoundError(f"{self._entity_name} not found: {item_id}") + del self._items[item_id] + await asyncio.to_thread(self._delete_item, item_id) + logger.info(f"Deleted {self._entity_name}: {item_id}") + + def count(self) -> int: + with self._lock: + return len(self._items) + + # -- Helpers ------------------------------------------------------------- + + def _check_name_unique(self, name: str, exclude_id: str = None) -> None: + """Raise ValueError if *name* is empty or already taken. + + Must be called while holding ``self._lock``. + """ + if not name or not name.strip(): + raise ValueError("Name is required") + for item_id, item in self._items.items(): + if item_id != exclude_id and getattr(item, "name", None) == name: + raise ValueError( + f"{self._entity_name} with name '{name}' already exists" + ) diff --git a/server/src/wled_controller/storage/color_strip_processing_template_store.py b/server/src/wled_controller/storage/color_strip_processing_template_store.py index ee12412..2a0652d 100644 --- a/server/src/wled_controller/storage/color_strip_processing_template_store.py +++ b/server/src/wled_controller/storage/color_strip_processing_template_store.py @@ -1,4 +1,4 @@ -"""Color strip processing template storage using JSON files.""" +"""Color strip processing template storage using SQLite.""" import uuid from datetime import datetime, timezone @@ -6,32 +6,33 @@ from typing import List, Optional from wled_controller.core.filters.filter_instance import FilterInstance from wled_controller.core.filters.registry import FilterRegistry -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore from wled_controller.storage.color_strip_processing_template import ColorStripProcessingTemplate +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) -class ColorStripProcessingTemplateStore(BaseJsonStore[ColorStripProcessingTemplate]): +class ColorStripProcessingTemplateStore(BaseSqliteStore[ColorStripProcessingTemplate]): """Storage for color strip processing templates. - All templates are persisted to the JSON file. + All templates are persisted to the database. On startup, if no templates exist, a default one is auto-created. """ - _json_key = "color_strip_processing_templates" + _table_name = "color_strip_processing_templates" _entity_name = "Color strip processing template" _version = "1.0.0" - def __init__(self, file_path: str): - super().__init__(file_path, ColorStripProcessingTemplate.from_dict) + def __init__(self, db: Database): + super().__init__(db, ColorStripProcessingTemplate.from_dict) self._ensure_initial_template() # Backward-compatible aliases - get_all_templates = BaseJsonStore.get_all - get_template = BaseJsonStore.get - delete_template = BaseJsonStore.delete + get_all_templates = BaseSqliteStore.get_all + get_template = BaseSqliteStore.get + delete_template = BaseSqliteStore.delete def _ensure_initial_template(self) -> None: """Auto-create a default color strip processing template if none exist.""" @@ -96,7 +97,7 @@ class ColorStripProcessingTemplateStore(BaseJsonStore[ColorStripProcessingTempla ) self._items[template_id] = template - self._save() + self._save_item(template_id, template) logger.info(f"Created color strip processing template: {name} ({template_id})") return template @@ -123,7 +124,7 @@ class ColorStripProcessingTemplateStore(BaseJsonStore[ColorStripProcessingTempla template.tags = tags template.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(template_id, template) logger.info(f"Updated color strip processing template: {template_id}") return template diff --git a/server/src/wled_controller/storage/color_strip_store.py b/server/src/wled_controller/storage/color_strip_store.py index 3721396..3bcca46 100644 --- a/server/src/wled_controller/storage/color_strip_store.py +++ b/server/src/wled_controller/storage/color_strip_store.py @@ -1,10 +1,11 @@ -"""Color strip source storage using JSON files.""" +"""Color strip source storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.utils import resolve_ref from wled_controller.storage.color_strip_source import ( ColorStripSource, @@ -17,18 +18,18 @@ from wled_controller.utils import get_logger logger = get_logger(__name__) -class ColorStripStore(BaseJsonStore[ColorStripSource]): +class ColorStripStore(BaseSqliteStore[ColorStripSource]): """Persistent storage for color strip sources.""" - _json_key = "color_strip_sources" + _table_name = "color_strip_sources" _entity_name = "Color strip source" - def __init__(self, file_path: str): - super().__init__(file_path, ColorStripSource.from_dict) + def __init__(self, db: Database): + super().__init__(db, ColorStripSource.from_dict) # Backward-compatible aliases - get_all_sources = BaseJsonStore.get_all - delete_source = BaseJsonStore.delete + get_all_sources = BaseSqliteStore.get_all + delete_source = BaseSqliteStore.delete def get_source(self, source_id: str) -> ColorStripSource: """Get a color strip source by ID (alias for get()).""" @@ -67,7 +68,7 @@ class ColorStripStore(BaseJsonStore[ColorStripSource]): ) self._items[source_id] = source - self._save() + self._save_item(source_id, source) logger.info(f"Created color strip source: {name} ({source_id}, type={source_type})") return source @@ -110,7 +111,7 @@ class ColorStripStore(BaseJsonStore[ColorStripSource]): source.apply_update(**kwargs) source.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(source_id, source) logger.info(f"Updated color strip source: {source_id}") return source diff --git a/server/src/wled_controller/storage/database.py b/server/src/wled_controller/storage/database.py new file mode 100644 index 0000000..1f2c591 --- /dev/null +++ b/server/src/wled_controller/storage/database.py @@ -0,0 +1,322 @@ +"""SQLite database connection wrapper. + +Provides a thread-safe, WAL-mode SQLite connection shared by all stores. +Each entity table uses the same schema: indexed columns for common queries +plus a JSON blob for the full entity data. +""" + +import json +import sqlite3 +import threading +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Dict, List, Tuple + +from wled_controller.utils import get_logger + +logger = get_logger(__name__) + +# When True, all database writes are suppressed. Set by the restore flow +# to prevent the old server process from overwriting freshly-restored data +# with stale in-memory state before the restart completes. +_writes_frozen = False + + +def freeze_writes() -> None: + """Block all database writes until the process exits (used after restore).""" + global _writes_frozen + _writes_frozen = True + logger.info("Database writes frozen - awaiting server restart") + + +def is_writes_frozen() -> bool: + """Check whether writes are currently frozen.""" + return _writes_frozen + + +# Schema version — bump when tables change +_SCHEMA_VERSION = 1 + +# All entity tables share this structure +_ENTITY_TABLES = [ + "devices", + "capture_templates", + "postprocessing_templates", + "picture_sources", + "output_targets", + "pattern_templates", + "color_strip_sources", + "audio_sources", + "audio_templates", + "value_sources", + "automations", + "scene_presets", + "sync_clocks", + "color_strip_processing_templates", + "gradients", + "weather_sources", +] + + +class Database: + """Thread-safe SQLite connection wrapper with WAL mode. + + All stores share a single Database instance. The connection uses + WAL journaling for concurrent read access and a single writer lock. + """ + + def __init__(self, db_path: str | Path): + self._path = Path(db_path) + self._path.parent.mkdir(parents=True, exist_ok=True) + self._conn = sqlite3.connect( + str(self._path), + check_same_thread=False, + ) + self._conn.row_factory = sqlite3.Row + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA busy_timeout=5000") + self._lock = threading.RLock() + + self._ensure_schema() + logger.info(f"Database opened: {self._path}") + + # -- Schema management --------------------------------------------------- + + def _ensure_schema(self) -> None: + """Create tables if they don't exist.""" + with self._lock: + # Schema version tracking + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS schema_version ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ) + """) + + # Key-value settings table + self._conn.execute(""" + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) + """) + + # Create entity tables + for table in _ENTITY_TABLES: + self._conn.execute(f""" + CREATE TABLE IF NOT EXISTS [{table}] ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + data TEXT NOT NULL + ) + """) + self._conn.execute( + f"CREATE INDEX IF NOT EXISTS idx_{table}_name ON [{table}](name)" + ) + + # Record schema version + existing = self._conn.execute( + "SELECT version FROM schema_version WHERE version = ?", + (_SCHEMA_VERSION,), + ).fetchone() + if not existing: + from datetime import datetime, timezone + self._conn.execute( + "INSERT OR IGNORE INTO schema_version (version, applied_at) VALUES (?, ?)", + (_SCHEMA_VERSION, datetime.now(timezone.utc).isoformat()), + ) + + self._conn.commit() + + # -- Low-level operations ------------------------------------------------ + + def execute(self, sql: str, params: Tuple = ()) -> sqlite3.Cursor: + """Execute a single SQL statement (auto-commits).""" + with self._lock: + cursor = self._conn.execute(sql, params) + self._conn.commit() + return cursor + + def execute_many(self, sql: str, params_list: List[Tuple]) -> None: + """Execute a parameterized statement for each params tuple.""" + with self._lock: + self._conn.executemany(sql, params_list) + self._conn.commit() + + @contextmanager + def transaction(self): + """Context manager for multi-statement transactions. + + Usage:: + + with db.transaction() as conn: + conn.execute("INSERT ...", (...)) + conn.execute("DELETE ...", (...)) + # auto-committed on exit, rolled back on exception + """ + with self._lock: + try: + yield self._conn + self._conn.commit() + except Exception: + self._conn.rollback() + raise + + # -- Entity helpers (used by BaseSqliteStore) ---------------------------- + + def load_all(self, table: str) -> List[Dict[str, Any]]: + """Load all rows from an entity table. + + Returns list of dicts parsed from the ``data`` JSON column. + """ + with self._lock: + rows = self._conn.execute( + f"SELECT id, data FROM [{table}]" + ).fetchall() + result = [] + for row in rows: + try: + item = json.loads(row["data"]) + result.append(item) + except json.JSONDecodeError as e: + logger.error(f"Corrupt JSON in {table}/{row['id']}: {e}") + return result + + def upsert(self, table: str, item_id: str, name: str, data: dict) -> None: + """Insert or replace a single entity row. + + Skipped silently when writes are frozen. + """ + if _writes_frozen: + return + json_data = json.dumps(data, ensure_ascii=False) + with self._lock: + self._conn.execute( + f"INSERT OR REPLACE INTO [{table}] (id, name, data) VALUES (?, ?, ?)", + (item_id, name, json_data), + ) + self._conn.commit() + + def delete_row(self, table: str, item_id: str) -> None: + """Delete a single entity row. + + Skipped silently when writes are frozen. + """ + if _writes_frozen: + return + with self._lock: + self._conn.execute( + f"DELETE FROM [{table}] WHERE id = ?", (item_id,) + ) + self._conn.commit() + + def delete_all(self, table: str) -> None: + """Delete all rows from an entity table. + + Skipped silently when writes are frozen. + """ + if _writes_frozen: + return + with self._lock: + self._conn.execute(f"DELETE FROM [{table}]") + self._conn.commit() + + def bulk_insert(self, table: str, items: List[Tuple[str, str, str]]) -> None: + """Bulk insert rows: list of (id, name, data_json) tuples. + + Skipped silently when writes are frozen. + """ + if _writes_frozen: + return + with self._lock: + self._conn.executemany( + f"INSERT OR REPLACE INTO [{table}] (id, name, data) VALUES (?, ?, ?)", + items, + ) + self._conn.commit() + + def count(self, table: str) -> int: + """Count rows in an entity table.""" + with self._lock: + row = self._conn.execute( + f"SELECT COUNT(*) as cnt FROM [{table}]" + ).fetchone() + return row["cnt"] + + def table_exists_with_data(self, table: str) -> bool: + """Check if a table exists and has at least one row.""" + with self._lock: + try: + row = self._conn.execute( + f"SELECT COUNT(*) as cnt FROM [{table}]" + ).fetchone() + return row["cnt"] > 0 + except sqlite3.OperationalError: + return False + + # -- Settings (key-value) ------------------------------------------------ + + def get_setting(self, key: str) -> dict | None: + """Read a setting by key. Returns parsed JSON dict, or None if not found.""" + with self._lock: + row = self._conn.execute( + "SELECT value FROM settings WHERE key = ?", (key,) + ).fetchone() + if row is None: + return None + try: + return json.loads(row["value"]) + except json.JSONDecodeError: + return None + + def set_setting(self, key: str, value: dict) -> None: + """Write a setting (upsert). Skipped when writes are frozen.""" + if _writes_frozen: + return + json_value = json.dumps(value, ensure_ascii=False) + with self._lock: + self._conn.execute( + "INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)", + (key, json_value), + ) + self._conn.commit() + + # -- Backup -------------------------------------------------------------- + + def backup_to(self, dest_path: str | Path) -> None: + """Create a consistent snapshot of the database using SQLite's backup API. + + Safe to call while the database is in use — SQLite handles locking. + """ + dest_path = Path(dest_path) + dest_path.parent.mkdir(parents=True, exist_ok=True) + with self._lock: + dest = sqlite3.connect(str(dest_path)) + try: + self._conn.backup(dest) + finally: + dest.close() + + def restore_from(self, src_path: str | Path) -> None: + """Replace the database contents from a backup file. + + The caller must restart the server after calling this — in-memory + caches in stores will be stale. + """ + src_path = Path(src_path) + if not src_path.exists(): + raise FileNotFoundError(f"Backup file not found: {src_path}") + with self._lock: + src = sqlite3.connect(str(src_path)) + try: + src.backup(self._conn) + finally: + src.close() + + # -- Lifecycle ----------------------------------------------------------- + + def close(self) -> None: + """Close the database connection.""" + with self._lock: + self._conn.close() + logger.info("Database connection closed") diff --git a/server/src/wled_controller/storage/device_store.py b/server/src/wled_controller/storage/device_store.py index 8e46eed..65cd4bb 100644 --- a/server/src/wled_controller/storage/device_store.py +++ b/server/src/wled_controller/storage/device_store.py @@ -1,12 +1,11 @@ -"""Device storage using JSON files.""" +"""Device storage using SQLite.""" -import json import uuid from datetime import datetime, timezone -from pathlib import Path from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.utils import get_logger logger = get_logger(__name__) @@ -190,14 +189,14 @@ _UPDATABLE_FIELDS: frozenset[str] = frozenset({ }) -class DeviceStore(BaseJsonStore[Device]): +class DeviceStore(BaseSqliteStore[Device]): """Persistent storage for WLED devices.""" - _json_key = "devices" + _table_name = "devices" _entity_name = "Device" - def __init__(self, storage_file: str | Path): - super().__init__(file_path=str(storage_file), deserializer=Device.from_dict) + def __init__(self, db: Database): + super().__init__(db, Device.from_dict) logger.info(f"Device store initialized with {len(self._items)} devices") # ── Backward-compat aliases ────────────────────────────────── @@ -278,7 +277,7 @@ class DeviceStore(BaseJsonStore[Device]): ) self._items[device_id] = device - self._save() + self._save_item(device_id, device) logger.info(f"Created device {device_id}: {name}") return device @@ -316,7 +315,7 @@ class DeviceStore(BaseJsonStore[Device]): new_device = Device(**device_fields) self._items[device_id] = new_device - self._save() + self._save_item(device_id, new_device) logger.info(f"Updated device {device_id}") return new_device @@ -330,15 +329,5 @@ class DeviceStore(BaseJsonStore[Device]): def clear(self): """Clear all devices (for testing).""" self._items.clear() - self._save() + self._db.delete_all(self._table_name) logger.warning("Cleared all devices from storage") - - def load_raw(self) -> dict: - """Load raw JSON data from storage (for migration).""" - if not self.file_path.exists(): - return {} - try: - with open(self.file_path, "r") as f: - return json.load(f) - except Exception: - return {} diff --git a/server/src/wled_controller/storage/gradient_store.py b/server/src/wled_controller/storage/gradient_store.py index b660ebf..b87da82 100644 --- a/server/src/wled_controller/storage/gradient_store.py +++ b/server/src/wled_controller/storage/gradient_store.py @@ -1,6 +1,6 @@ """Gradient storage with built-in seeding. -Provides CRUD for gradient entities. On first run (empty/missing file), +Provides CRUD for gradient entities. On first run (empty/missing data), seeds 8 built-in gradients matching the legacy hardcoded palettes. Built-in gradients are read-only and cannot be deleted or modified. """ @@ -9,7 +9,8 @@ import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.gradient import Gradient from wled_controller.utils import get_logger @@ -43,12 +44,12 @@ def _tuples_to_stops(tuples: list) -> list: return [{"position": t[0], "color": [t[1], t[2], t[3]]} for t in tuples] -class GradientStore(BaseJsonStore[Gradient]): - _json_key = "gradients" +class GradientStore(BaseSqliteStore[Gradient]): + _table_name = "gradients" _entity_name = "Gradient" - def __init__(self, file_path: str): - super().__init__(file_path, Gradient.from_dict) + def __init__(self, db: Database): + super().__init__(db, Gradient.from_dict) if not self._items: self._seed_builtins() @@ -70,7 +71,7 @@ class GradientStore(BaseJsonStore[Gradient]): logger.info(f"Seeded {len(_BUILTIN_DEFS)} built-in gradients") # Aliases - get_all_gradients = BaseJsonStore.get_all + get_all_gradients = BaseSqliteStore.get_all def get_gradient(self, gradient_id: str) -> Gradient: return self.get(gradient_id) @@ -104,7 +105,7 @@ class GradientStore(BaseJsonStore[Gradient]): tags=tags or [], ) self._items[gid] = gradient - self._save() + self._save_item(gid, gradient) logger.info(f"Created gradient: {name} ({gid})") return gradient @@ -129,7 +130,7 @@ class GradientStore(BaseJsonStore[Gradient]): if tags is not None: gradient.tags = tags gradient.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(gradient_id, gradient) logger.info(f"Updated gradient: {gradient_id}") return gradient diff --git a/server/src/wled_controller/storage/output_target_store.py b/server/src/wled_controller/storage/output_target_store.py index 1d697f3..323a222 100644 --- a/server/src/wled_controller/storage/output_target_store.py +++ b/server/src/wled_controller/storage/output_target_store.py @@ -1,10 +1,11 @@ -"""Output target storage using JSON files.""" +"""Output target storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.output_target import OutputTarget from wled_controller.storage.wled_output_target import WledOutputTarget from wled_controller.storage.key_colors_output_target import ( @@ -18,20 +19,19 @@ logger = get_logger(__name__) DEFAULT_STATE_CHECK_INTERVAL = 30 # seconds -class OutputTargetStore(BaseJsonStore[OutputTarget]): +class OutputTargetStore(BaseSqliteStore[OutputTarget]): """Persistent storage for output targets.""" - _json_key = "output_targets" + _table_name = "output_targets" _entity_name = "Output target" - _legacy_json_keys = ["picture_targets"] - def __init__(self, file_path: str): - super().__init__(file_path, OutputTarget.from_dict) + def __init__(self, db: Database): + super().__init__(db, OutputTarget.from_dict) # Backward-compatible aliases - get_all_targets = BaseJsonStore.get_all - get_target = BaseJsonStore.get - delete_target = BaseJsonStore.delete + get_all_targets = BaseSqliteStore.get_all + get_target = BaseSqliteStore.get + delete_target = BaseSqliteStore.delete def create_target( self, @@ -101,7 +101,7 @@ class OutputTargetStore(BaseJsonStore[OutputTarget]): target.tags = tags or [] self._items[target_id] = target - self._save() + self._save_item(target_id, target) logger.info(f"Created output target: {name} ({target_id}, type={target_type})") return target @@ -156,7 +156,7 @@ class OutputTargetStore(BaseJsonStore[OutputTarget]): ) target.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(target_id, target) logger.info(f"Updated output target: {target_id}") return target diff --git a/server/src/wled_controller/storage/pattern_template_store.py b/server/src/wled_controller/storage/pattern_template_store.py index abf85ab..9189fc3 100644 --- a/server/src/wled_controller/storage/pattern_template_store.py +++ b/server/src/wled_controller/storage/pattern_template_store.py @@ -1,10 +1,11 @@ -"""Pattern template storage using JSON files.""" +"""Pattern template storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.key_colors_output_target import KeyColorRectangle from wled_controller.storage.pattern_template import PatternTemplate from wled_controller.utils import get_logger @@ -12,24 +13,24 @@ from wled_controller.utils import get_logger logger = get_logger(__name__) -class PatternTemplateStore(BaseJsonStore[PatternTemplate]): +class PatternTemplateStore(BaseSqliteStore[PatternTemplate]): """Storage for pattern templates (rectangle layouts for key color extraction). - All templates are persisted to the JSON file. + All templates are persisted to the database. On startup, if no templates exist, a default one is auto-created. """ - _json_key = "pattern_templates" + _table_name = "pattern_templates" _entity_name = "Pattern template" - def __init__(self, file_path: str): - super().__init__(file_path, PatternTemplate.from_dict) + def __init__(self, db: Database): + super().__init__(db, PatternTemplate.from_dict) self._ensure_initial_template() # Backward-compatible aliases - get_all_templates = BaseJsonStore.get_all - get_template = BaseJsonStore.get - delete_template = BaseJsonStore.delete + get_all_templates = BaseSqliteStore.get_all + get_template = BaseSqliteStore.get + delete_template = BaseSqliteStore.delete def _ensure_initial_template(self) -> None: """Auto-create a default pattern template if none exist.""" @@ -80,7 +81,7 @@ class PatternTemplateStore(BaseJsonStore[PatternTemplate]): ) self._items[template_id] = template - self._save() + self._save_item(template_id, template) logger.info(f"Created pattern template: {name} ({template_id})") return template @@ -106,7 +107,7 @@ class PatternTemplateStore(BaseJsonStore[PatternTemplate]): template.tags = tags template.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(template_id, template) logger.info(f"Updated pattern template: {template_id}") return template diff --git a/server/src/wled_controller/storage/picture_source_store.py b/server/src/wled_controller/storage/picture_source_store.py index b756044..06221e6 100644 --- a/server/src/wled_controller/storage/picture_source_store.py +++ b/server/src/wled_controller/storage/picture_source_store.py @@ -1,10 +1,11 @@ -"""Picture source storage using JSON files.""" +"""Picture source storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional, Set -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.utils import resolve_ref from wled_controller.storage.picture_source import ( PictureSource, @@ -18,26 +19,26 @@ from wled_controller.utils import get_logger logger = get_logger(__name__) -class PictureSourceStore(BaseJsonStore[PictureSource]): +class PictureSourceStore(BaseSqliteStore[PictureSource]): """Storage for picture sources. Supports raw and processed stream types with cycle detection for processed streams that reference other streams. """ - _json_key = "picture_sources" + _table_name = "picture_sources" _entity_name = "Picture source" - def __init__(self, file_path: str): - super().__init__(file_path, PictureSource.from_dict) + def __init__(self, db: Database): + super().__init__(db, PictureSource.from_dict) # Backward-compatible aliases - get_all_sources = BaseJsonStore.get_all - get_source = BaseJsonStore.get + get_all_sources = BaseSqliteStore.get_all + get_source = BaseSqliteStore.get # Legacy aliases (old code used "stream" naming) - get_all_streams = BaseJsonStore.get_all - get_stream = BaseJsonStore.get + get_all_streams = BaseSqliteStore.get_all + get_stream = BaseSqliteStore.get # ── Helpers ─────────────────────────────────────────────────────── @@ -171,7 +172,7 @@ class PictureSourceStore(BaseJsonStore[PictureSource]): ) self._items[stream_id] = stream - self._save() + self._save_item(stream_id, stream) logger.info(f"Created picture source: {name} ({stream_id}, type={stream_type})") return stream @@ -255,7 +256,7 @@ class PictureSourceStore(BaseJsonStore[PictureSource]): stream.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(stream_id, stream) logger.info(f"Updated picture source: {stream_id}") return stream @@ -278,7 +279,7 @@ class PictureSourceStore(BaseJsonStore[PictureSource]): ) del self._items[stream_id] - self._save() + self._delete_item(stream_id) logger.info(f"Deleted picture source: {stream_id}") diff --git a/server/src/wled_controller/storage/postprocessing_template_store.py b/server/src/wled_controller/storage/postprocessing_template_store.py index c60044b..44a9071 100644 --- a/server/src/wled_controller/storage/postprocessing_template_store.py +++ b/server/src/wled_controller/storage/postprocessing_template_store.py @@ -1,4 +1,4 @@ -"""Postprocessing template storage using JSON files.""" +"""Postprocessing template storage using SQLite.""" import uuid from datetime import datetime, timezone @@ -6,7 +6,8 @@ from typing import List, Optional from wled_controller.core.filters.filter_instance import FilterInstance from wled_controller.core.filters.registry import FilterRegistry -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.picture_source import ProcessedPictureSource from wled_controller.storage.postprocessing_template import PostprocessingTemplate from wled_controller.utils import get_logger @@ -14,25 +15,25 @@ from wled_controller.utils import get_logger logger = get_logger(__name__) -class PostprocessingTemplateStore(BaseJsonStore[PostprocessingTemplate]): +class PostprocessingTemplateStore(BaseSqliteStore[PostprocessingTemplate]): """Storage for postprocessing templates. - All templates are persisted to the JSON file. + All templates are persisted to the database. On startup, if no templates exist, a default one is auto-created. """ - _json_key = "postprocessing_templates" + _table_name = "postprocessing_templates" _entity_name = "Postprocessing template" _version = "2.0.0" - def __init__(self, file_path: str): - super().__init__(file_path, PostprocessingTemplate.from_dict) + def __init__(self, db: Database): + super().__init__(db, PostprocessingTemplate.from_dict) self._ensure_initial_template() # Backward-compatible aliases - get_all_templates = BaseJsonStore.get_all - get_template = BaseJsonStore.get - delete_template = BaseJsonStore.delete + get_all_templates = BaseSqliteStore.get_all + get_template = BaseSqliteStore.get + delete_template = BaseSqliteStore.delete def _ensure_initial_template(self) -> None: """Auto-create a default postprocessing template if none exist.""" @@ -90,7 +91,7 @@ class PostprocessingTemplateStore(BaseJsonStore[PostprocessingTemplate]): ) self._items[template_id] = template - self._save() + self._save_item(template_id, template) logger.info(f"Created postprocessing template: {name} ({template_id})") return template @@ -120,7 +121,7 @@ class PostprocessingTemplateStore(BaseJsonStore[PostprocessingTemplate]): template.tags = tags template.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(template_id, template) logger.info(f"Updated postprocessing template: {template_id}") return template diff --git a/server/src/wled_controller/storage/scene_preset_store.py b/server/src/wled_controller/storage/scene_preset_store.py index accce06..6d1d544 100644 --- a/server/src/wled_controller/storage/scene_preset_store.py +++ b/server/src/wled_controller/storage/scene_preset_store.py @@ -1,27 +1,28 @@ -"""Scene preset storage using JSON files.""" +"""Scene preset storage using SQLite.""" from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.scene_preset import ScenePreset, TargetSnapshot from wled_controller.utils import get_logger logger = get_logger(__name__) -class ScenePresetStore(BaseJsonStore[ScenePreset]): +class ScenePresetStore(BaseSqliteStore[ScenePreset]): """Persistent storage for scene presets.""" - _json_key = "scene_presets" + _table_name = "scene_presets" _entity_name = "Scene preset" - def __init__(self, file_path: str): - super().__init__(file_path, ScenePreset.from_dict) + def __init__(self, db: Database): + super().__init__(db, ScenePreset.from_dict) # Backward-compatible aliases - get_preset = BaseJsonStore.get - delete_preset = BaseJsonStore.delete + get_preset = BaseSqliteStore.get + delete_preset = BaseSqliteStore.delete def get_all_presets(self) -> List[ScenePreset]: """Get all presets sorted by order field.""" @@ -35,7 +36,7 @@ class ScenePresetStore(BaseJsonStore[ScenePreset]): self._check_name_unique(preset.name) self._items[preset.id] = preset - self._save() + self._save_item(preset.id, preset) logger.info(f"Created scene preset: {preset.name} ({preset.id})") return preset @@ -63,7 +64,7 @@ class ScenePresetStore(BaseJsonStore[ScenePreset]): preset.tags = tags preset.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(preset_id, preset) logger.info(f"Updated scene preset: {preset_id}") return preset @@ -73,6 +74,6 @@ class ScenePresetStore(BaseJsonStore[ScenePreset]): existing.targets = preset.targets existing.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(preset_id, existing) logger.info(f"Recaptured scene preset: {preset_id}") return existing diff --git a/server/src/wled_controller/storage/sync_clock_store.py b/server/src/wled_controller/storage/sync_clock_store.py index eacc090..e66b72c 100644 --- a/server/src/wled_controller/storage/sync_clock_store.py +++ b/server/src/wled_controller/storage/sync_clock_store.py @@ -1,27 +1,28 @@ -"""Synchronization clock storage using JSON files.""" +"""Synchronization clock storage.""" import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.sync_clock import SyncClock from wled_controller.utils import get_logger logger = get_logger(__name__) -class SyncClockStore(BaseJsonStore[SyncClock]): - _json_key = "sync_clocks" +class SyncClockStore(BaseSqliteStore[SyncClock]): + _table_name = "sync_clocks" _entity_name = "Sync clock" - def __init__(self, file_path: str): - super().__init__(file_path, SyncClock.from_dict) + def __init__(self, db: Database): + super().__init__(db, SyncClock.from_dict) # Backward-compatible aliases - get_all_clocks = BaseJsonStore.get_all - get_clock = BaseJsonStore.get - delete_clock = BaseJsonStore.delete + get_all_clocks = BaseSqliteStore.get_all + get_clock = BaseSqliteStore.get + delete_clock = BaseSqliteStore.delete def create_clock( self, @@ -45,7 +46,7 @@ class SyncClockStore(BaseJsonStore[SyncClock]): ) self._items[cid] = clock - self._save() + self._save_item(cid, clock) logger.info(f"Created sync clock: {name} ({cid}, speed={clock.speed})") return clock @@ -70,6 +71,6 @@ class SyncClockStore(BaseJsonStore[SyncClock]): clock.tags = tags clock.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(clock_id, clock) logger.info(f"Updated sync clock: {clock_id}") return clock diff --git a/server/src/wled_controller/storage/template_store.py b/server/src/wled_controller/storage/template_store.py index 602ef64..675d276 100644 --- a/server/src/wled_controller/storage/template_store.py +++ b/server/src/wled_controller/storage/template_store.py @@ -1,36 +1,37 @@ -"""Template storage using JSON files.""" +"""Template storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import Any, Dict, List, Optional from wled_controller.core.capture_engines.factory import EngineRegistry -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.template import CaptureTemplate from wled_controller.utils import get_logger logger = get_logger(__name__) -class TemplateStore(BaseJsonStore[CaptureTemplate]): +class TemplateStore(BaseSqliteStore[CaptureTemplate]): """Storage for capture templates. - All templates are persisted to the JSON file. + All templates are persisted to the database. On startup, if no templates exist, one is auto-created using the highest-priority available engine. """ - _json_key = "templates" + _table_name = "capture_templates" _entity_name = "Capture template" - def __init__(self, file_path: str): - super().__init__(file_path, CaptureTemplate.from_dict) + def __init__(self, db: Database): + super().__init__(db, CaptureTemplate.from_dict) self._ensure_initial_template() # Backward-compatible aliases - get_all_templates = BaseJsonStore.get_all - get_template = BaseJsonStore.get - delete_template = BaseJsonStore.delete + get_all_templates = BaseSqliteStore.get_all + get_template = BaseSqliteStore.get + delete_template = BaseSqliteStore.delete def _ensure_initial_template(self) -> None: """Auto-create a template if none exist, using the best available engine.""" @@ -85,7 +86,7 @@ class TemplateStore(BaseJsonStore[CaptureTemplate]): ) self._items[template_id] = template - self._save() + self._save_item(template_id, template) logger.info(f"Created template: {name} ({template_id})") return template @@ -114,7 +115,7 @@ class TemplateStore(BaseJsonStore[CaptureTemplate]): template.tags = tags template.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(template_id, template) logger.info(f"Updated template: {template_id}") return template diff --git a/server/src/wled_controller/storage/value_source_store.py b/server/src/wled_controller/storage/value_source_store.py index 9baa031..09d9f87 100644 --- a/server/src/wled_controller/storage/value_source_store.py +++ b/server/src/wled_controller/storage/value_source_store.py @@ -1,10 +1,11 @@ -"""Value source storage using JSON files.""" +"""Value source storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.utils import resolve_ref from wled_controller.storage.value_source import ( AdaptiveValueSource, @@ -19,19 +20,19 @@ from wled_controller.utils import get_logger logger = get_logger(__name__) -class ValueSourceStore(BaseJsonStore[ValueSource]): +class ValueSourceStore(BaseSqliteStore[ValueSource]): """Persistent storage for value sources.""" - _json_key = "value_sources" + _table_name = "value_sources" _entity_name = "Value source" - def __init__(self, file_path: str): - super().__init__(file_path, ValueSource.from_dict) + def __init__(self, db: Database): + super().__init__(db, ValueSource.from_dict) # Backward-compatible aliases - get_all_sources = BaseJsonStore.get_all - get_source = BaseJsonStore.get - delete_source = BaseJsonStore.delete + get_all_sources = BaseSqliteStore.get_all + get_source = BaseSqliteStore.get + delete_source = BaseSqliteStore.delete # ── CRUD ───────────────────────────────────────────────────────── @@ -128,7 +129,7 @@ class ValueSourceStore(BaseJsonStore[ValueSource]): ) self._items[sid] = source - self._save() + self._save_item(sid, source) logger.info(f"Created value source: {name} ({sid}, type={source_type})") return source @@ -223,7 +224,7 @@ class ValueSourceStore(BaseJsonStore[ValueSource]): source.max_value = max_value source.updated_at = datetime.now(timezone.utc) - self._save() + self._save_item(source_id, source) logger.info(f"Updated value source: {source_id}") return source diff --git a/server/src/wled_controller/storage/weather_source_store.py b/server/src/wled_controller/storage/weather_source_store.py index c91bccd..7855e67 100644 --- a/server/src/wled_controller/storage/weather_source_store.py +++ b/server/src/wled_controller/storage/weather_source_store.py @@ -1,29 +1,30 @@ -"""Weather source storage using JSON files.""" +"""Weather source storage using SQLite.""" import uuid from datetime import datetime, timezone from typing import List, Optional -from wled_controller.storage.base_store import BaseJsonStore +from wled_controller.storage.base_sqlite_store import BaseSqliteStore +from wled_controller.storage.database import Database from wled_controller.storage.weather_source import WeatherSource from wled_controller.utils import get_logger logger = get_logger(__name__) -class WeatherSourceStore(BaseJsonStore[WeatherSource]): +class WeatherSourceStore(BaseSqliteStore[WeatherSource]): """Persistent storage for weather sources.""" - _json_key = "weather_sources" + _table_name = "weather_sources" _entity_name = "Weather source" - def __init__(self, file_path: str): - super().__init__(file_path, WeatherSource.from_dict) + def __init__(self, db: Database): + super().__init__(db, WeatherSource.from_dict) # Backward-compatible aliases - get_all_sources = BaseJsonStore.get_all - get_source = BaseJsonStore.get - delete_source = BaseJsonStore.delete + get_all_sources = BaseSqliteStore.get_all + get_source = BaseSqliteStore.get + delete_source = BaseSqliteStore.delete def create_source( self, @@ -67,7 +68,7 @@ class WeatherSourceStore(BaseJsonStore[WeatherSource]): ) self._items[sid] = source - self._save() + self._save_item(sid, source) logger.info(f"Created weather source: {name} ({sid})") return source @@ -115,6 +116,6 @@ class WeatherSourceStore(BaseJsonStore[WeatherSource]): ) self._items[source_id] = updated - self._save() + self._save_item(source_id, updated) logger.info(f"Updated weather source: {updated.name} ({source_id})") return updated diff --git a/server/src/wled_controller/templates/modals/settings.html b/server/src/wled_controller/templates/modals/settings.html index efef66b..69da896 100644 --- a/server/src/wled_controller/templates/modals/settings.html +++ b/server/src/wled_controller/templates/modals/settings.html @@ -92,47 +92,11 @@ - + -
-
- - -
- - -
- - -
- -
- - -
- - - -
-
@@ -164,7 +128,10 @@
- +
+ + +