feat: migrate storage from JSON files to SQLite
Some checks failed
Lint & Test / test (push) Failing after 28s

Replace 22 individual JSON store files with a single SQLite database
(data/ledgrab.db). All entity stores now use BaseSqliteStore backed by
SQLite with WAL mode, write-through caching, and thread-safe access.

- Add Database class with SQLite backup/restore API
- Add BaseSqliteStore as drop-in replacement for BaseJsonStore
- Convert all 16 entity stores to SQLite
- Move global settings (MQTT, external URL, auto-backup) to SQLite
  settings table
- Replace JSON backup/restore with SQLite snapshot backups (.db files)
- Remove partial export/import feature (backend + frontend)
- Update demo seed to write directly to SQLite
- Add "Backup Now" button to settings UI
- Remove StorageConfig file path fields (single database_file remains)
This commit is contained in:
2026-03-25 00:03:19 +03:00
parent 29fb944494
commit 9dfd2365f4
38 changed files with 941 additions and 880 deletions

View File

@@ -41,7 +41,7 @@ from wled_controller.core.mqtt.mqtt_service import MQTTService
from wled_controller.core.devices.mqtt_client import set_mqtt_service
from wled_controller.core.backup.auto_backup import AutoBackupEngine
from wled_controller.core.processing.os_notification_listener import OsNotificationListener
from wled_controller.api.routes.system import STORE_MAP
from wled_controller.storage.database import Database
from wled_controller.utils import setup_logging, get_logger, install_broadcast_handler
# Initialize logging
@@ -52,29 +52,32 @@ logger = get_logger(__name__)
# Get configuration
config = get_config()
# Seed demo data before stores are loaded (first-run only)
# Initialize SQLite database
db = Database(config.storage.database_file)
# Seed demo data after DB is ready (first-run only)
if config.demo:
from wled_controller.core.demo_seed import seed_demo_data
seed_demo_data(config.storage)
seed_demo_data(db)
# Initialize storage and processing
device_store = DeviceStore(config.storage.devices_file)
template_store = TemplateStore(config.storage.templates_file)
pp_template_store = PostprocessingTemplateStore(config.storage.postprocessing_templates_file)
picture_source_store = PictureSourceStore(config.storage.picture_sources_file)
output_target_store = OutputTargetStore(config.storage.output_targets_file)
pattern_template_store = PatternTemplateStore(config.storage.pattern_templates_file)
color_strip_store = ColorStripStore(config.storage.color_strip_sources_file)
audio_source_store = AudioSourceStore(config.storage.audio_sources_file)
audio_template_store = AudioTemplateStore(config.storage.audio_templates_file)
value_source_store = ValueSourceStore(config.storage.value_sources_file)
automation_store = AutomationStore(config.storage.automations_file)
scene_preset_store = ScenePresetStore(config.storage.scene_presets_file)
sync_clock_store = SyncClockStore(config.storage.sync_clocks_file)
cspt_store = ColorStripProcessingTemplateStore(config.storage.color_strip_processing_templates_file)
gradient_store = GradientStore(config.storage.gradients_file)
device_store = DeviceStore(db)
template_store = TemplateStore(db)
pp_template_store = PostprocessingTemplateStore(db)
picture_source_store = PictureSourceStore(db)
output_target_store = OutputTargetStore(db)
pattern_template_store = PatternTemplateStore(db)
color_strip_store = ColorStripStore(db)
audio_source_store = AudioSourceStore(db)
audio_template_store = AudioTemplateStore(db)
value_source_store = ValueSourceStore(db)
automation_store = AutomationStore(db)
scene_preset_store = ScenePresetStore(db)
sync_clock_store = SyncClockStore(db)
cspt_store = ColorStripProcessingTemplateStore(db)
gradient_store = GradientStore(db)
gradient_store.migrate_palette_references(color_strip_store)
weather_source_store = WeatherSourceStore(config.storage.weather_sources_file)
weather_source_store = WeatherSourceStore(db)
sync_clock_manager = SyncClockManager(sync_clock_store)
weather_manager = WeatherManager(weather_source_store)
@@ -156,34 +159,18 @@ async def lifespan(app: FastAPI):
device_store=device_store,
)
# Create auto-backup engine — derive paths from storage config so that
# Create auto-backup engine — derive paths from database location so that
# demo mode auto-backups go to data/demo/ instead of data/.
_data_dir = Path(config.storage.devices_file).parent
_data_dir = Path(config.storage.database_file).parent
auto_backup_engine = AutoBackupEngine(
settings_path=_data_dir / "auto_backup_settings.json",
backup_dir=_data_dir / "backups",
store_map=STORE_MAP,
storage_config=config.storage,
db=db,
)
# Verify STORE_MAP covers all StorageConfig file fields.
# Catches missed additions early (at startup) rather than silently
# excluding new stores from backups.
storage_attrs = {
attr for attr in config.storage.model_fields
if attr.endswith("_file")
}
mapped_attrs = set(STORE_MAP.values())
unmapped = storage_attrs - mapped_attrs
if unmapped:
logger.warning(
f"StorageConfig fields not in STORE_MAP (missing from backups): "
f"{sorted(unmapped)}"
)
# Initialize API dependencies
init_dependencies(
device_store, template_store, processor_manager,
database=db,
pp_template_store=pp_template_store,
pattern_template_store=pattern_template_store,
picture_source_store=picture_source_store,