feat: migrate storage from JSON files to SQLite
Some checks failed
Lint & Test / test (push) Failing after 28s
Some checks failed
Lint & Test / test (push) Failing after 28s
Replace 22 individual JSON store files with a single SQLite database (data/ledgrab.db). All entity stores now use BaseSqliteStore backed by SQLite with WAL mode, write-through caching, and thread-safe access. - Add Database class with SQLite backup/restore API - Add BaseSqliteStore as drop-in replacement for BaseJsonStore - Convert all 16 entity stores to SQLite - Move global settings (MQTT, external URL, auto-backup) to SQLite settings table - Replace JSON backup/restore with SQLite snapshot backups (.db files) - Remove partial export/import feature (backend + frontend) - Update demo seed to write directly to SQLite - Add "Backup Now" button to settings UI - Remove StorageConfig file path fields (single database_file remains)
This commit is contained in:
@@ -1,23 +1,20 @@
|
||||
"""System routes: backup, restore, export, import, auto-backup.
|
||||
"""System routes: backup, restore, auto-backup.
|
||||
|
||||
Extracted from system.py to keep files under 800 lines.
|
||||
All backups are SQLite database snapshots (.db files).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
|
||||
from fastapi.responses import StreamingResponse
|
||||
|
||||
from wled_controller import __version__
|
||||
from wled_controller.api.auth import AuthRequired
|
||||
from wled_controller.api.dependencies import get_auto_backup_engine
|
||||
from wled_controller.api.dependencies import get_auto_backup_engine, get_database
|
||||
from wled_controller.api.schemas.system import (
|
||||
AutoBackupSettings,
|
||||
AutoBackupStatusResponse,
|
||||
@@ -26,38 +23,13 @@ from wled_controller.api.schemas.system import (
|
||||
RestoreResponse,
|
||||
)
|
||||
from wled_controller.core.backup.auto_backup import AutoBackupEngine
|
||||
from wled_controller.config import get_config
|
||||
from wled_controller.storage.base_store import freeze_saves
|
||||
from wled_controller.utils import atomic_write_json, get_logger
|
||||
from wled_controller.storage.database import Database, freeze_writes
|
||||
from wled_controller.utils import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Configuration backup / restore
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Mapping: logical store name -> StorageConfig attribute name
|
||||
STORE_MAP = {
|
||||
"devices": "devices_file",
|
||||
"capture_templates": "templates_file",
|
||||
"postprocessing_templates": "postprocessing_templates_file",
|
||||
"picture_sources": "picture_sources_file",
|
||||
"output_targets": "output_targets_file",
|
||||
"pattern_templates": "pattern_templates_file",
|
||||
"color_strip_sources": "color_strip_sources_file",
|
||||
"audio_sources": "audio_sources_file",
|
||||
"audio_templates": "audio_templates_file",
|
||||
"value_sources": "value_sources_file",
|
||||
"sync_clocks": "sync_clocks_file",
|
||||
"color_strip_processing_templates": "color_strip_processing_templates_file",
|
||||
"automations": "automations_file",
|
||||
"scene_presets": "scene_presets_file",
|
||||
"gradients": "gradients_file",
|
||||
"weather_sources": "weather_sources_file",
|
||||
}
|
||||
|
||||
_SERVER_DIR = Path(__file__).resolve().parents[4]
|
||||
|
||||
|
||||
@@ -82,147 +54,74 @@ def _schedule_restart() -> None:
|
||||
threading.Thread(target=_restart, daemon=True).start()
|
||||
|
||||
|
||||
@router.get("/api/v1/system/export/{store_key}", tags=["System"])
|
||||
def export_store(store_key: str, _: AuthRequired):
|
||||
"""Download a single entity store as a JSON file."""
|
||||
if store_key not in STORE_MAP:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}",
|
||||
)
|
||||
config = get_config()
|
||||
file_path = Path(getattr(config.storage, STORE_MAP[store_key]))
|
||||
if file_path.exists():
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
else:
|
||||
data = {}
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backup / restore (SQLite snapshots)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
export = {
|
||||
"meta": {
|
||||
"format": "ledgrab-partial-export",
|
||||
"format_version": 1,
|
||||
"store_key": store_key,
|
||||
"app_version": __version__,
|
||||
"created_at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||
},
|
||||
"store": data,
|
||||
}
|
||||
content = json.dumps(export, indent=2, ensure_ascii=False)
|
||||
|
||||
@router.get("/api/v1/system/backup", tags=["System"])
|
||||
def backup_config(_: AuthRequired, db: Database = Depends(get_database)):
|
||||
"""Download a full database backup as a .db file."""
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
|
||||
try:
|
||||
db.backup_to(tmp_path)
|
||||
content = tmp_path.read_bytes()
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
from datetime import datetime, timezone
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S")
|
||||
filename = f"ledgrab-{store_key}-{timestamp}.json"
|
||||
filename = f"ledgrab-backup-{timestamp}.db"
|
||||
|
||||
return StreamingResponse(
|
||||
io.BytesIO(content.encode("utf-8")),
|
||||
media_type="application/json",
|
||||
io.BytesIO(content),
|
||||
media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/v1/system/import/{store_key}", tags=["System"])
|
||||
async def import_store(
|
||||
store_key: str,
|
||||
@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"])
|
||||
async def restore_config(
|
||||
_: AuthRequired,
|
||||
file: UploadFile = File(...),
|
||||
merge: bool = Query(False, description="Merge into existing data instead of replacing"),
|
||||
db: Database = Depends(get_database),
|
||||
):
|
||||
"""Upload a partial export file to replace or merge one entity store. Triggers server restart."""
|
||||
if store_key not in STORE_MAP:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"Unknown store '{store_key}'. Valid keys: {sorted(STORE_MAP.keys())}",
|
||||
)
|
||||
"""Upload a .db backup file to restore all configuration. Triggers server restart."""
|
||||
raw = await file.read()
|
||||
if len(raw) > 50 * 1024 * 1024: # 50 MB limit
|
||||
raise HTTPException(status_code=400, detail="Backup file too large (max 50 MB)")
|
||||
|
||||
if len(raw) < 100:
|
||||
raise HTTPException(status_code=400, detail="File too small to be a valid SQLite database")
|
||||
|
||||
# SQLite files start with "SQLite format 3\000"
|
||||
if not raw[:16].startswith(b"SQLite format 3"):
|
||||
raise HTTPException(status_code=400, detail="Not a valid SQLite database file")
|
||||
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile(suffix=".db", delete=False) as tmp:
|
||||
tmp.write(raw)
|
||||
tmp_path = Path(tmp.name)
|
||||
|
||||
try:
|
||||
raw = await file.read()
|
||||
if len(raw) > 10 * 1024 * 1024:
|
||||
raise HTTPException(status_code=400, detail="File too large (max 10 MB)")
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid JSON: {e}")
|
||||
def _restore():
|
||||
db.restore_from(tmp_path)
|
||||
|
||||
# Support both full-backup format and partial-export format
|
||||
if "stores" in payload and isinstance(payload.get("meta"), dict):
|
||||
# Full backup: extract the specific store
|
||||
if payload["meta"].get("format") not in ("ledgrab-backup",):
|
||||
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file")
|
||||
stores = payload.get("stores", {})
|
||||
if store_key not in stores:
|
||||
raise HTTPException(status_code=400, detail=f"Backup does not contain store '{store_key}'")
|
||||
incoming = stores[store_key]
|
||||
elif isinstance(payload.get("meta"), dict) and payload["meta"].get("format") == "ledgrab-partial-export":
|
||||
# Partial export format
|
||||
if payload["meta"].get("store_key") != store_key:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"File is for store '{payload['meta']['store_key']}', not '{store_key}'",
|
||||
)
|
||||
incoming = payload.get("store", {})
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup or partial export file")
|
||||
await asyncio.to_thread(_restore)
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
|
||||
if not isinstance(incoming, dict):
|
||||
raise HTTPException(status_code=400, detail="Store data must be a JSON object")
|
||||
|
||||
config = get_config()
|
||||
file_path = Path(getattr(config.storage, STORE_MAP[store_key]))
|
||||
|
||||
def _write():
|
||||
if merge and file_path.exists():
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
existing = json.load(f)
|
||||
if isinstance(existing, dict):
|
||||
existing.update(incoming)
|
||||
atomic_write_json(file_path, existing)
|
||||
return len(existing)
|
||||
atomic_write_json(file_path, incoming)
|
||||
return len(incoming)
|
||||
|
||||
count = await asyncio.to_thread(_write)
|
||||
freeze_saves()
|
||||
logger.info(f"Imported store '{store_key}' ({count} entries, merge={merge}). Scheduling restart...")
|
||||
freeze_writes()
|
||||
logger.info("Database restored from uploaded backup. Scheduling restart...")
|
||||
_schedule_restart()
|
||||
return {
|
||||
"status": "imported",
|
||||
"store_key": store_key,
|
||||
"entries": count,
|
||||
"merge": merge,
|
||||
"restart_scheduled": True,
|
||||
"message": f"Imported {count} entries for '{store_key}'. Server restarting...",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/v1/system/backup", tags=["System"])
|
||||
def backup_config(_: AuthRequired):
|
||||
"""Download all configuration as a single JSON backup file."""
|
||||
config = get_config()
|
||||
stores = {}
|
||||
for store_key, config_attr in STORE_MAP.items():
|
||||
file_path = Path(getattr(config.storage, config_attr))
|
||||
if file_path.exists():
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
stores[store_key] = json.load(f)
|
||||
else:
|
||||
stores[store_key] = {}
|
||||
|
||||
backup = {
|
||||
"meta": {
|
||||
"format": "ledgrab-backup",
|
||||
"format_version": 1,
|
||||
"app_version": __version__,
|
||||
"created_at": datetime.now(timezone.utc).isoformat() + "Z",
|
||||
"store_count": len(stores),
|
||||
},
|
||||
"stores": stores,
|
||||
}
|
||||
|
||||
content = json.dumps(backup, indent=2, ensure_ascii=False)
|
||||
timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H%M%S")
|
||||
filename = f"ledgrab-backup-{timestamp}.json"
|
||||
|
||||
return StreamingResponse(
|
||||
io.BytesIO(content.encode("utf-8")),
|
||||
media_type="application/json",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
return RestoreResponse(
|
||||
status="restored",
|
||||
restart_scheduled=True,
|
||||
message="Database restored from backup. Server restarting...",
|
||||
)
|
||||
|
||||
|
||||
@@ -237,110 +136,12 @@ def restart_server(_: AuthRequired):
|
||||
|
||||
@router.post("/api/v1/system/shutdown", tags=["System"])
|
||||
def shutdown_server(_: AuthRequired):
|
||||
"""Gracefully shut down the server.
|
||||
|
||||
Signals uvicorn to exit, which triggers the lifespan shutdown handler
|
||||
(persists all stores to disk, stops processors, etc.).
|
||||
Used by the restart script to ensure data is saved before the process exits.
|
||||
"""
|
||||
"""Gracefully shut down the server."""
|
||||
from wled_controller.server_ref import request_shutdown
|
||||
request_shutdown()
|
||||
return {"status": "shutting_down"}
|
||||
|
||||
|
||||
@router.post("/api/v1/system/restore", response_model=RestoreResponse, tags=["System"])
|
||||
async def restore_config(
|
||||
_: AuthRequired,
|
||||
file: UploadFile = File(...),
|
||||
):
|
||||
"""Upload a backup file to restore all configuration. Triggers server restart."""
|
||||
# Read and parse
|
||||
try:
|
||||
raw = await file.read()
|
||||
if len(raw) > 10 * 1024 * 1024: # 10 MB limit
|
||||
raise HTTPException(status_code=400, detail="Backup file too large (max 10 MB)")
|
||||
backup = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid JSON file: {e}")
|
||||
|
||||
# Validate envelope
|
||||
meta = backup.get("meta")
|
||||
if not isinstance(meta, dict) or meta.get("format") != "ledgrab-backup":
|
||||
raise HTTPException(status_code=400, detail="Not a valid LED Grab backup file")
|
||||
|
||||
fmt_version = meta.get("format_version", 0)
|
||||
if fmt_version > 1:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Backup format version {fmt_version} is not supported by this server version",
|
||||
)
|
||||
|
||||
stores = backup.get("stores")
|
||||
if not isinstance(stores, dict):
|
||||
raise HTTPException(status_code=400, detail="Backup file missing 'stores' section")
|
||||
|
||||
known_keys = set(STORE_MAP.keys())
|
||||
present_keys = known_keys & set(stores.keys())
|
||||
if not present_keys:
|
||||
raise HTTPException(status_code=400, detail="Backup contains no recognized store data")
|
||||
|
||||
for key in present_keys:
|
||||
if not isinstance(stores[key], dict):
|
||||
raise HTTPException(status_code=400, detail=f"Store '{key}' in backup is not a valid JSON object")
|
||||
|
||||
# Guard: reject backups where every store is empty (version key only, no entities).
|
||||
# This prevents accidental data wipes from restoring a backup taken when the
|
||||
# server had no data loaded.
|
||||
total_entities = 0
|
||||
for key in present_keys:
|
||||
store_data = stores[key]
|
||||
for field_key, field_val in store_data.items():
|
||||
if field_key != "version" and isinstance(field_val, dict):
|
||||
total_entities += len(field_val)
|
||||
if total_entities == 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Backup contains no entity data (all stores are empty). Aborting to prevent data loss.",
|
||||
)
|
||||
|
||||
# Log missing stores as warnings
|
||||
missing = known_keys - present_keys
|
||||
if missing:
|
||||
for store_key in sorted(missing):
|
||||
logger.warning(f"Restore: backup is missing store '{store_key}' — existing data will be kept")
|
||||
|
||||
# Write store files atomically (in thread to avoid blocking event loop)
|
||||
config = get_config()
|
||||
|
||||
def _write_stores():
|
||||
count = 0
|
||||
for store_key, config_attr in STORE_MAP.items():
|
||||
if store_key in stores:
|
||||
file_path = Path(getattr(config.storage, config_attr))
|
||||
atomic_write_json(file_path, stores[store_key])
|
||||
count += 1
|
||||
logger.info(f"Restored store: {store_key} -> {file_path}")
|
||||
return count
|
||||
|
||||
written = await asyncio.to_thread(_write_stores)
|
||||
|
||||
# Freeze all store saves so the old process can't overwrite restored files
|
||||
# with stale in-memory data before the restart completes.
|
||||
freeze_saves()
|
||||
|
||||
logger.info(f"Restore complete: {written}/{len(STORE_MAP)} stores written. Scheduling restart...")
|
||||
_schedule_restart()
|
||||
|
||||
return RestoreResponse(
|
||||
status="restored",
|
||||
stores_written=written,
|
||||
stores_total=len(STORE_MAP),
|
||||
missing_stores=sorted(missing) if missing else [],
|
||||
restart_scheduled=True,
|
||||
message=f"Restored {written} stores. Server restarting...",
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auto-backup settings & saved backups
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -419,7 +220,7 @@ def download_saved_backup(
|
||||
content = path.read_bytes()
|
||||
return StreamingResponse(
|
||||
io.BytesIO(content),
|
||||
media_type="application/json",
|
||||
media_type="application/octet-stream",
|
||||
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user