feat: migrate storage from JSON files to SQLite
Some checks failed
Lint & Test / test (push) Failing after 28s

Replace 22 individual JSON store files with a single SQLite database
(data/ledgrab.db). All entity stores now use BaseSqliteStore backed by
SQLite with WAL mode, write-through caching, and thread-safe access.

- Add Database class with SQLite backup/restore API
- Add BaseSqliteStore as drop-in replacement for BaseJsonStore
- Convert all 16 entity stores to SQLite
- Move global settings (MQTT, external URL, auto-backup) to SQLite
  settings table
- Replace JSON backup/restore with SQLite snapshot backups (.db files)
- Remove partial export/import feature (backend + frontend)
- Update demo seed to write directly to SQLite
- Add "Backup Now" button to settings UI
- Remove StorageConfig file path fields (single database_file remains)
This commit is contained in:
2026-03-25 00:03:19 +03:00
parent 29fb944494
commit 9dfd2365f4
38 changed files with 941 additions and 880 deletions

View File

@@ -1,23 +1,20 @@
"""System routes: backup, restore, export, import, auto-backup.
"""System routes: backup, restore, auto-backup.
Extracted from system.py to keep files under 800 lines.
All backups are SQLite database snapshots (.db files).
"""
import asyncio
import io
import json
import subprocess
import sys
import threading
from datetime import datetime, timezone
from pathlib import Path
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.responses import StreamingResponse
from wled_controller import __version__
from wled_controller.api.auth import AuthRequired
from wled_controller.api.dependencies import get_auto_backup_engine
from wled_controller.api.dependencies import get_auto_backup_engine, get_database
from wled_controller.api.schemas.system import (
AutoBackupSettings,
AutoBackupStatusResponse,
@@ -26,38 +23,13 @@ from wled_controller.api.schemas.system import (
RestoreResponse,
)
from wled_controller.core.backup.auto_backup import AutoBackupEngine
from wled_controller.config import get_config
from wled_controller.storage.base_store import freeze_saves
from wled_controller.utils import atomic_write_json, get_logger
from wled_controller.storage.database import Database, freeze_writes
from wled_controller.utils import get_logger
logger = get_logger(__name__)
router = APIRouter()
# ---------------------------------------------------------------------------
# Configuration backup / restore
# ---------------------------------------------------------------------------
# Mapping: logical store name -> StorageConfig attribute name
STORE_MAP = {
"devices": "devices_file",
"capture_templates": "templates_file",
"postprocessing_templates": "postprocessing_templates_file",
"picture_sources": "picture_sources_file",
"output_targets": "output_targets_file",
"pattern_templates": "pattern_templates_file",
"color_strip_sources": "color_strip_sources_file",
"audio_sources": "audio_sources_file",
"audio_templates": "audio_templates_file",
"value_sources": "value_sources_file",
"sync_clocks": "sync_clocks_file",
"color_strip_processing_templates": "color_strip_processing_templates_file",
"automations": "automations_file",
"scene_presets": "scene_presets_file",
"gradients": "gradients_file",
"weather_sources": "weather_sources_file",
}
_SERVER_DIR = Path(__file__).resolve().parents[4]
@@ -82,147 +54,74 @@ def _schedule_restart() -> None:
threading.Thread(target=_restart, daemon=True).start()
@router.get("/api/v1/system/export/{store_key}", tags=["System"])
def export_store(store_key: str, _: AuthRequired):
"""Download a single entity store as a JSON file."""
if store_key not in STORE_MAP:
raise HTTPException(
status_code=404,
detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}",
)
config = get_config()
file_path = Path(getattr(config.storage, STORE_MAP[store_key]))
if file_path.exists():
with open(file_path, "r", encoding="utf-8") as f:
data = json.load(f)
else:
data = {}
# ---------------------------------------------------------------------------
# Backup / restore (SQLite snapshots)
# ---------------------------------------------------------------------------
export = {
"meta": {
"format": "ledgrab-partial-export",
"format_version": 1,
"store_key": store_key,
"app_version": __version__,
"created_at": datetime.now(timezone.utc).isoformat() + "Z",
},
"store": data,
}
content = json.dumps(export, indent=2, ensure_ascii=False)
@router.get("/api/v1/system/backup", tags=["System"])
def backup_config(_: AuthRequired, db: Database = Depends(get_database)):
"""Download a full database backup as a .db file."""
import tempfile
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
tmp_path = Path(tmp.name)
try:
db.backup_to(tmp_path)
content = tmp_path.read_bytes()
finally:
tmp_path.unlink(missing_ok=True)
from datetime import datetime, timezone
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S")
filename = f"ledgrab-{store_key}-{timestamp}.json"
filename = f"ledgrab-backup-{timestamp}.db"
return StreamingResponse(
io.BytesIO(content.encode("utf-8")),
media_type="application/json",
io.BytesIO(content),
media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.post("/api/v1/system/import/{store_key}", tags=["System"])
async def import_store(
store_key: str,
@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"])
async def restore_config(
_: AuthRequired,
file: UploadFile = File(...),
merge: bool = Query(False, description="Merge into existing data instead of replacing"),
db: Database = Depends(get_database),
):
"""Upload a partial export file to replace or merge one entity store. Triggers server restart."""
if store_key not in STORE_MAP:
raise HTTPException(
status_code=404,
detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}",
)
"""Upload a .db backup file to restore all configuration. Triggers server restart."""
raw = await file.read()
if len(raw) > 50 * 1024 * 1024: # 50 MB limit
raise HTTPException(status_code=400, detail="Backup file too large (max 50 MB)")
if len(raw) < 100:
raise HTTPException(status_code=400, detail="File too small to be a valid SQLite database")
# SQLite files start with "SQLite format 3\000"
if not raw[:16].startswith(b"SQLite format 3"):
raise HTTPException(status_code=400, detail="Not a valid SQLite database file")
import tempfile
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
tmp.write(raw)
tmp_path = Path(tmp.name)
try:
raw = await file.read()
if len(raw) > 10 * 1024 * 1024:
raise HTTPException(status_code=400, detail="File too large (max 10 MB)")
payload = json.loads(raw)
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}")
def _restore():
db.restore_from(tmp_path)
# Support both full-backup format and partial-export format
if "stores" in payload and isinstance(payload.get("meta"), dict):
# Full backup: extract the specific store
if payload["meta"].get("format") not in ("ledgrab-backup",):
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file")
stores = payload.get("stores", {})
if store_key not in stores:
raise HTTPException(status_code=400, detail=f"Backup does not contain store '{store_key}'")
incoming = stores[store_key]
elif isinstance(payload.get("meta"), dict) and payload["meta"].get("format") == "ledgrab-partial-export":
# Partial export format
if payload["meta"].get("store_key") != store_key:
raise HTTPException(
status_code=400,
detail=f"File is for store '{payload['meta']['store_key']}', not '{store_key}'",
)
incoming = payload.get("store", {})
else:
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file")
await asyncio.to_thread(_restore)
finally:
tmp_path.unlink(missing_ok=True)
if not isinstance(incoming, dict):
raise HTTPException(status_code=400, detail="Store data must be a JSON object")
config = get_config()
file_path = Path(getattr(config.storage, STORE_MAP[store_key]))
def _write():
if merge and file_path.exists():
with open(file_path, "r", encoding="utf-8") as f:
existing = json.load(f)
if isinstance(existing, dict):
existing.update(incoming)
atomic_write_json(file_path, existing)
return len(existing)
atomic_write_json(file_path, incoming)
return len(incoming)
count = await asyncio.to_thread(_write)
freeze_saves()
logger.info(f"Imported store '{store_key}' ({count} entries, merge={merge}). Scheduling restart...")
freeze_writes()
logger.info("Database restored from uploaded backup. Scheduling restart...")
_schedule_restart()
return {
"status": "imported",
"store_key": store_key,
"entries": count,
"merge": merge,
"restart_scheduled": True,
"message": f"Imported {count} entries for '{store_key}'. Server restarting...",
}
@router.get("/api/v1/system/backup", tags=["System"])
def backup_config(_: AuthRequired):
"""Download all configuration as a single JSON backup file."""
config = get_config()
stores = {}
for store_key, config_attr in STORE_MAP.items():
file_path = Path(getattr(config.storage, config_attr))
if file_path.exists():
with open(file_path, "r", encoding="utf-8") as f:
stores[store_key] = json.load(f)
else:
stores[store_key] = {}
backup = {
"meta": {
"format": "ledgrab-backup",
"format_version": 1,
"app_version": __version__,
"created_at": datetime.now(timezone.utc).isoformat() + "Z",
"store_count": len(stores),
},
"stores": stores,
}
content = json.dumps(backup, indent=2, ensure_ascii=False)
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S")
filename = f"ledgrab-backup-{timestamp}.json"
return StreamingResponse(
io.BytesIO(content.encode("utf-8")),
media_type="application/json",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
return RestoreResponse(
status="restored",
restart_scheduled=True,
message="Database restored from backup. Server restarting...",
)
@@ -237,110 +136,12 @@ def restart_server(_: AuthRequired):
@router.post("/api/v1/system/shutdown", tags=["System"])
def shutdown_server(_: AuthRequired):
"""Gracefully shut down the server.
Signals uvicorn to exit, which triggers the lifespan shutdown handler
(persists all stores to disk, stops processors, etc.).
Used by the restart script to ensure data is saved before the process exits.
"""
"""Gracefully shut down the server."""
from wled_controller.server_ref import request_shutdown
request_shutdown()
return {"status": "shutting_down"}
@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"])
async def restore_config(
_: AuthRequired,
file: UploadFile = File(...),
):
"""Upload a backup file to restore all configuration. Triggers server restart."""
# Read and parse
try:
raw = await file.read()
if len(raw) > 10 * 1024 * 1024: # 10 MB limit
raise HTTPException(status_code=400, detail="Backup file too large (max 10 MB)")
backup = json.loads(raw)
except json.JSONDecodeError as e:
raise HTTPException(status_code=400, detail=f"Invalid JSON file: {e}")
# Validate envelope
meta = backup.get("meta")
if not isinstance(meta, dict) or meta.get("format") != "ledgrab-backup":
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup file")
fmt_version = meta.get("format_version", 0)
if fmt_version > 1:
raise HTTPException(
status_code=400,
detail=f"Backup format version {fmt_version} is not supported by this server version",
)
stores = backup.get("stores")
if not isinstance(stores, dict):
raise HTTPException(status_code=400, detail="Backup file missing 'stores' section")
known_keys = set(STORE_MAP.keys())
present_keys = known_keys & set(stores.keys())
if not present_keys:
raise HTTPException(status_code=400, detail="Backup contains no recognized store data")
for key in present_keys:
if not isinstance(stores[key], dict):
raise HTTPException(status_code=400, detail=f"Store '{key}' in backup is not a valid JSON object")
# Guard: reject backups where every store is empty (version key only, no entities).
# This prevents accidental data wipes from restoring a backup taken when the
# server had no data loaded.
total_entities = 0
for key in present_keys:
store_data = stores[key]
for field_key, field_val in store_data.items():
if field_key != "version" and isinstance(field_val, dict):
total_entities += len(field_val)
if total_entities == 0:
raise HTTPException(
status_code=400,
detail="Backup contains no entity data (all stores are empty). Aborting to prevent data loss.",
)
# Log missing stores as warnings
missing = known_keys - present_keys
if missing:
for store_key in sorted(missing):
logger.warning(f"Restore: backup is missing store '{store_key}' — existing data will be kept")
# Write store files atomically (in thread to avoid blocking event loop)
config = get_config()
def _write_stores():
count = 0
for store_key, config_attr in STORE_MAP.items():
if store_key in stores:
file_path = Path(getattr(config.storage, config_attr))
atomic_write_json(file_path, stores[store_key])
count += 1
logger.info(f"Restored store: {store_key} -> {file_path}")
return count
written = await asyncio.to_thread(_write_stores)
# Freeze all store saves so the old process can't overwrite restored files
# with stale in-memory data before the restart completes.
freeze_saves()
logger.info(f"Restore complete: {written}/{len(STORE_MAP)} stores written. Scheduling restart...")
_schedule_restart()
return RestoreResponse(
status="restored",
stores_written=written,
stores_total=len(STORE_MAP),
missing_stores=sorted(missing) if missing else [],
restart_scheduled=True,
message=f"Restored {written} stores. Server restarting...",
)
# ---------------------------------------------------------------------------
# Auto-backup settings & saved backups
# ---------------------------------------------------------------------------
@@ -419,7 +220,7 @@ def download_saved_backup(
content = path.read_bytes()
return StreamingResponse(
io.BytesIO(content),
media_type="application/json",
media_type="application/octet-stream",
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)

View File

@@ -45,8 +45,7 @@ from wled_controller.core.capture.screen_capture import get_available_displays
from wled_controller.utils import get_logger
from wled_controller.storage.base_store import EntityNotFoundError
# Re-export STORE_MAP and load_external_url so existing callers still work
from wled_controller.api.routes.backup import STORE_MAP # noqa: F401
# Re-export load_external_url so existing callers still work
from wled_controller.api.routes.system_settings import load_external_url # noqa: F401
logger = get_logger(__name__)

View File

@@ -4,15 +4,14 @@ Extracted from system.py to keep files under 800 lines.
"""
import asyncio
import json
import logging
import re
from pathlib import Path
from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketDisconnect
from fastapi import APIRouter, Depends, HTTPException, Query, WebSocket, WebSocketDisconnect
from pydantic import BaseModel
from wled_controller.api.auth import AuthRequired
from wled_controller.api.dependencies import get_database
from wled_controller.api.schemas.system import (
ExternalUrlRequest,
ExternalUrlResponse,
@@ -22,6 +21,7 @@ from wled_controller.api.schemas.system import (
MQTTSettingsResponse,
)
from wled_controller.config import get_config
from wled_controller.storage.database import Database
from wled_controller.utils import get_logger
logger = get_logger(__name__)
@@ -33,21 +33,9 @@ router = APIRouter()
# MQTT settings
# ---------------------------------------------------------------------------
_MQTT_SETTINGS_FILE: Path | None = None
def _get_mqtt_settings_path() -> Path:
global _MQTT_SETTINGS_FILE
if _MQTT_SETTINGS_FILE is None:
cfg = get_config()
# Derive the data directory from any known storage file path
data_dir = Path(cfg.storage.devices_file).parent
_MQTT_SETTINGS_FILE = data_dir / "mqtt_settings.json"
return _MQTT_SETTINGS_FILE
def _load_mqtt_settings() -> dict:
"""Load MQTT settings: YAML config defaults overridden by JSON overrides file."""
def _load_mqtt_settings(db: Database) -> dict:
"""Load MQTT settings: YAML config defaults overridden by DB settings."""
cfg = get_config()
defaults = {
"enabled": cfg.mqtt.enabled,
@@ -58,31 +46,20 @@ def _load_mqtt_settings() -> dict:
"client_id": cfg.mqtt.client_id,
"base_topic": cfg.mqtt.base_topic,
}
path = _get_mqtt_settings_path()
if path.exists():
try:
with open(path, "r", encoding="utf-8") as f:
overrides = json.load(f)
defaults.update(overrides)
except Exception as e:
logger.warning(f"Failed to load MQTT settings override file: {e}")
overrides = db.get_setting("mqtt")
if overrides:
defaults.update(overrides)
return defaults
def _save_mqtt_settings(settings: dict) -> None:
"""Persist MQTT settings to the JSON override file."""
from wled_controller.utils import atomic_write_json
atomic_write_json(_get_mqtt_settings_path(), settings)
@router.get(
"/api/v1/system/mqtt/settings",
response_model=MQTTSettingsResponse,
tags=["System"],
)
async def get_mqtt_settings(_: AuthRequired):
async def get_mqtt_settings(_: AuthRequired, db: Database = Depends(get_database)):
"""Get current MQTT broker settings. Password is masked."""
s = _load_mqtt_settings()
s = _load_mqtt_settings(db)
return MQTTSettingsResponse(
enabled=s["enabled"],
broker_host=s["broker_host"],
@@ -99,9 +76,9 @@ async def get_mqtt_settings(_: AuthRequired):
response_model=MQTTSettingsResponse,
tags=["System"],
)
async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest):
async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest, db: Database = Depends(get_database)):
"""Update MQTT broker settings. If password is empty string, the existing password is preserved."""
current = _load_mqtt_settings()
current = _load_mqtt_settings(db)
# If caller sends an empty password, keep the existing one
password = body.password if body.password else current.get("password", "")
@@ -115,7 +92,7 @@ async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest):
"client_id": body.client_id,
"base_topic": body.base_topic,
}
_save_mqtt_settings(new_settings)
db.set_setting("mqtt", new_settings)
logger.info("MQTT settings updated")
return MQTTSettingsResponse(
@@ -133,44 +110,25 @@ async def update_mqtt_settings(_: AuthRequired, body: MQTTSettingsRequest):
# External URL setting
# ---------------------------------------------------------------------------
_EXTERNAL_URL_FILE: Path | None = None
def _get_external_url_path() -> Path:
global _EXTERNAL_URL_FILE
if _EXTERNAL_URL_FILE is None:
cfg = get_config()
data_dir = Path(cfg.storage.devices_file).parent
_EXTERNAL_URL_FILE = data_dir / "external_url.json"
return _EXTERNAL_URL_FILE
def load_external_url() -> str:
def load_external_url(db: Database | None = None) -> str:
"""Load the external URL setting. Returns empty string if not set."""
path = _get_external_url_path()
if path.exists():
try:
with open(path, "r", encoding="utf-8") as f:
data = json.load(f)
return data.get("external_url", "")
except Exception:
pass
if db is None:
from wled_controller.api.dependencies import get_database
db = get_database()
data = db.get_setting("external_url")
if data:
return data.get("external_url", "")
return ""
def _save_external_url(url: str) -> None:
from wled_controller.utils import atomic_write_json
atomic_write_json(_get_external_url_path(), {"external_url": url})
@router.get(
"/api/v1/system/external-url",
response_model=ExternalUrlResponse,
tags=["System"],
)
async def get_external_url(_: AuthRequired):
async def get_external_url(_: AuthRequired, db: Database = Depends(get_database)):
"""Get the configured external base URL."""
return ExternalUrlResponse(external_url=load_external_url())
return ExternalUrlResponse(external_url=load_external_url(db))
@router.put(
@@ -178,10 +136,10 @@ async def get_external_url(_: AuthRequired):
response_model=ExternalUrlResponse,
tags=["System"],
)
async def update_external_url(_: AuthRequired, body: ExternalUrlRequest):
async def update_external_url(_: AuthRequired, body: ExternalUrlRequest, db: Database = Depends(get_database)):
"""Set the external base URL used in webhook URLs and other user-visible URLs."""
url = body.external_url.strip().rstrip("/")
_save_external_url(url)
db.set_setting("external_url", {"external_url": url})
logger.info("External URL updated: %s", url or "(cleared)")
return ExternalUrlResponse(external_url=url)