feat: migrate storage from JSON files to SQLite
Some checks failed
Lint & Test / test (push) Failing after 28s
Some checks failed
Lint & Test / test (push) Failing after 28s
Replace 22 individual JSON store files with a single SQLite database (data/ledgrab.db). All entity stores now use BaseSqliteStore backed by SQLite with WAL mode, write-through caching, and thread-safe access. - Add Database class with SQLite backup/restore API - Add BaseSqliteStore as drop-in replacement for BaseJsonStore - Convert all 16 entity stores to SQLite - Move global settings (MQTT, external URL, auto-backup) to SQLite settings table - Replace JSON backup/restore with SQLite snapshot backups (.db files) - Remove partial export/import feature (backend + frontend) - Update demo seed to write directly to SQLite - Add "Backup Now" button to settings UI - Remove StorageConfig file path fields (single database_file remains)
This commit is contained in:
@@ -41,7 +41,7 @@ from wled_controller.core.mqtt.mqtt_service import MQTTService
|
||||
from wled_controller.core.devices.mqtt_client import set_mqtt_service
|
||||
from wled_controller.core.backup.auto_backup import AutoBackupEngine
|
||||
from wled_controller.core.processing.os_notification_listener import OsNotificationListener
|
||||
from wled_controller.api.routes.system import STORE_MAP
|
||||
from wled_controller.storage.database import Database
|
||||
from wled_controller.utils import setup_logging, get_logger, install_broadcast_handler
|
||||
|
||||
# Initialize logging
|
||||
@@ -52,29 +52,32 @@ logger = get_logger(__name__)
|
||||
# Get configuration
|
||||
config = get_config()
|
||||
|
||||
# Seed demo data before stores are loaded (first-run only)
|
||||
# Initialize SQLite database
|
||||
db = Database(config.storage.database_file)
|
||||
|
||||
# Seed demo data after DB is ready (first-run only)
|
||||
if config.demo:
|
||||
from wled_controller.core.demo_seed import seed_demo_data
|
||||
seed_demo_data(config.storage)
|
||||
seed_demo_data(db)
|
||||
|
||||
# Initialize storage and processing
|
||||
device_store = DeviceStore(config.storage.devices_file)
|
||||
template_store = TemplateStore(config.storage.templates_file)
|
||||
pp_template_store = PostprocessingTemplateStore(config.storage.postprocessing_templates_file)
|
||||
picture_source_store = PictureSourceStore(config.storage.picture_sources_file)
|
||||
output_target_store = OutputTargetStore(config.storage.output_targets_file)
|
||||
pattern_template_store = PatternTemplateStore(config.storage.pattern_templates_file)
|
||||
color_strip_store = ColorStripStore(config.storage.color_strip_sources_file)
|
||||
audio_source_store = AudioSourceStore(config.storage.audio_sources_file)
|
||||
audio_template_store = AudioTemplateStore(config.storage.audio_templates_file)
|
||||
value_source_store = ValueSourceStore(config.storage.value_sources_file)
|
||||
automation_store = AutomationStore(config.storage.automations_file)
|
||||
scene_preset_store = ScenePresetStore(config.storage.scene_presets_file)
|
||||
sync_clock_store = SyncClockStore(config.storage.sync_clocks_file)
|
||||
cspt_store = ColorStripProcessingTemplateStore(config.storage.color_strip_processing_templates_file)
|
||||
gradient_store = GradientStore(config.storage.gradients_file)
|
||||
device_store = DeviceStore(db)
|
||||
template_store = TemplateStore(db)
|
||||
pp_template_store = PostprocessingTemplateStore(db)
|
||||
picture_source_store = PictureSourceStore(db)
|
||||
output_target_store = OutputTargetStore(db)
|
||||
pattern_template_store = PatternTemplateStore(db)
|
||||
color_strip_store = ColorStripStore(db)
|
||||
audio_source_store = AudioSourceStore(db)
|
||||
audio_template_store = AudioTemplateStore(db)
|
||||
value_source_store = ValueSourceStore(db)
|
||||
automation_store = AutomationStore(db)
|
||||
scene_preset_store = ScenePresetStore(db)
|
||||
sync_clock_store = SyncClockStore(db)
|
||||
cspt_store = ColorStripProcessingTemplateStore(db)
|
||||
gradient_store = GradientStore(db)
|
||||
gradient_store.migrate_palette_references(color_strip_store)
|
||||
weather_source_store = WeatherSourceStore(config.storage.weather_sources_file)
|
||||
weather_source_store = WeatherSourceStore(db)
|
||||
sync_clock_manager = SyncClockManager(sync_clock_store)
|
||||
weather_manager = WeatherManager(weather_source_store)
|
||||
|
||||
@@ -156,34 +159,18 @@ async def lifespan(app: FastAPI):
|
||||
device_store=device_store,
|
||||
)
|
||||
|
||||
# Create auto-backup engine — derive paths from storage config so that
|
||||
# Create auto-backup engine — derive paths from database location so that
|
||||
# demo mode auto-backups go to data/demo/ instead of data/.
|
||||
_data_dir = Path(config.storage.devices_file).parent
|
||||
_data_dir = Path(config.storage.database_file).parent
|
||||
auto_backup_engine = AutoBackupEngine(
|
||||
settings_path=_data_dir / "auto_backup_settings.json",
|
||||
backup_dir=_data_dir / "backups",
|
||||
store_map=STORE_MAP,
|
||||
storage_config=config.storage,
|
||||
db=db,
|
||||
)
|
||||
|
||||
# Verify STORE_MAP covers all StorageConfig file fields.
|
||||
# Catches missed additions early (at startup) rather than silently
|
||||
# excluding new stores from backups.
|
||||
storage_attrs = {
|
||||
attr for attr in config.storage.model_fields
|
||||
if attr.endswith("_file")
|
||||
}
|
||||
mapped_attrs = set(STORE_MAP.values())
|
||||
unmapped = storage_attrs - mapped_attrs
|
||||
if unmapped:
|
||||
logger.warning(
|
||||
f"StorageConfig fields not in STORE_MAP (missing from backups): "
|
||||
f"{sorted(unmapped)}"
|
||||
)
|
||||
|
||||
# Initialize API dependencies
|
||||
init_dependencies(
|
||||
device_store, template_store, processor_manager,
|
||||
database=db,
|
||||
pp_template_store=pp_template_store,
|
||||
pattern_template_store=pattern_template_store,
|
||||
picture_source_store=picture_source_store,
|
||||
|
||||
Reference in New Issue
Block a user