feat: production-readiness hardening across security, async, DB, ops
Security - SSRF: async DNS resolver; allow_redirects=False on all outbound clients; matrix homeserver_url validated on create/update/test; update_provider and email_bot merge incoming config and reject ***-masked secrets. - Auth: bcrypt offloaded to asyncio.to_thread; JWT now carries iss/aud + leeway and rejects missing claims; setup TOCTOU closed inside a transaction; rate limits extended (default 600/min, 10/min on password change, 30/min on needs-setup); constant-time login to prevent username enumeration. - Config: rejects known dev secret keys; validates CORS origin schemes, port range, token lifetimes. - Webhook handlers stream-read body with a 1 MiB cap; Discord 429 retries bounded (3 attempts, Retry-After capped at 60 s). - CSP + HSTS added to SecurityHeadersMiddleware. Async / runtime - SQLite engine: WAL, synchronous=NORMAL, foreign_keys=ON, busy_timeout, pool_pre_ping, dispose on shutdown. - Lifespan shutdown now stops scheduler before closing HTTP session and disposing the engine. - Shared aiohttp session locked against concurrent first-caller races; core NotificationDispatcher accepts and reuses it. - Storage and scheduled backup writes wrapped in asyncio.to_thread. - NUT client writes bounded by asyncio.wait_for. - Telegram poller switched from 3 s short-poll to 30 s interval + 25 s long-poll (~10x fewer API calls). Database - New performance-indexes migration covers every FK/owner column and hot-path composite (notification_tracker(provider_id, enabled); event_log(user_id, created_at DESC); webhook_payload_log(provider_id, created_at DESC); action_execution(action_id, started_at DESC)). - New schema_version table for future upgrade gating. - __system__ placeholder user (id=0) seeded so user_id=0 system defaults satisfy the newly enforced FK; filtered out of /auth/needs-setup, /api/users, and setup. - list_notification_trackers rewritten to batched loads (was 1+N+N*M). - Retention job extended to event_log, webhook_payload_log, and action_execution; retention days exposed as a setting. Scheduler - AsyncIOScheduler job_defaults: coalesce, misfire_grace_time=300, max_instances=1. Ops - uvicorn runs with proxy_headers, forwarded_allow_ips, timeout_graceful_shutdown; access log suppressed in non-debug. - FastAPI version string now reads from importlib.metadata. - New /api/ready endpoint separate from /api/health. - docker-compose drops the ALLOW_PRIVATE_URLS=1 default, adds mem/cpu/pid limits, read_only + tmpfs, cap_drop:ALL, no-new-privileges; healthcheck targets /api/ready. - CI now runs on push/PR with backend pytest, frontend svelte-check + build, and a non-push image build; release workflow gated on tests, publishes immutable sha-<commit> image tag, adds Trivy scan. Tests - New packages/server/tests/ with 29 passing tests: config validation, JWT round-trip + aud/alg=none rejection, SSRF scheme and private-range enforcement (sync + async), Discord bounded retry, and a lifespan-level /api/health + /api/ready smoke check. - Renamed the misnamed services/test_dispatch.py to manual_dispatch.py so pytest never auto-collects production code. Frontend - /login now redirects already-authenticated users to /, shows a distinct 'backend unreachable' banner (en/ru) when /auth/needs-setup fails.
This commit is contained in:
@@ -0,0 +1,450 @@
|
||||
"""Test dispatch — manual trigger through the real NotificationDispatcher.
|
||||
|
||||
No separate logic — just builds a ServiceEvent + TargetConfig from DB
|
||||
objects and dispatches through the same path the watcher uses.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from notify_bridge_core.models.events import EventType, ServiceEvent
|
||||
from notify_bridge_core.models.media import MediaAsset
|
||||
from notify_bridge_core.notifications.dispatcher import NotificationDispatcher, TargetConfig
|
||||
from notify_bridge_core.providers.base import ServiceProviderType
|
||||
|
||||
from ..database.models import (
|
||||
NotificationTarget,
|
||||
NotificationTracker,
|
||||
NotificationTrackerTarget,
|
||||
ServiceProvider,
|
||||
TemplateConfig,
|
||||
TemplateSlot,
|
||||
TrackingConfig,
|
||||
)
|
||||
from .dispatch_helpers import _resolve_target
|
||||
from .watcher import _get_telegram_caches
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Maps test_type → DB template slot name
|
||||
_TEST_TYPE_SLOT_MAP = {
|
||||
"periodic": "periodic_summary_message",
|
||||
"scheduled": "scheduled_assets_message",
|
||||
"memory": "memory_mode_message",
|
||||
}
|
||||
|
||||
|
||||
async def dispatch_test_notification(
|
||||
*,
|
||||
session: AsyncSession,
|
||||
tracker: NotificationTracker,
|
||||
tt: NotificationTrackerTarget,
|
||||
target: NotificationTarget,
|
||||
test_type: str,
|
||||
locale: str = "en",
|
||||
) -> dict[str, Any]:
|
||||
"""Dispatch a test notification through the real NotificationDispatcher."""
|
||||
|
||||
# Load provider
|
||||
provider = await session.get(ServiceProvider, tracker.provider_id)
|
||||
if not provider:
|
||||
return {"success": False, "error": "Provider not found"}
|
||||
provider_config = dict(provider.config)
|
||||
collection_ids = list(tracker.collection_ids or [])
|
||||
|
||||
# Resolve tracking config: per-link override, else the tracker's default.
|
||||
# The real watcher applies this fallback in ``load_link_data`` — tests
|
||||
# must use the same logic or the user's per-tracker defaults look broken.
|
||||
tracking_config_id = tt.tracking_config_id or tracker.default_tracking_config_id
|
||||
tracking_config = None
|
||||
if tracking_config_id:
|
||||
tracking_config = await session.get(TrackingConfig, tracking_config_id)
|
||||
|
||||
# Same fallback for template config.
|
||||
template_config_id = tt.template_config_id or tracker.default_template_config_id
|
||||
template_config = None
|
||||
template_slots: dict[str, dict[str, str]] | None = None
|
||||
slot_name = _TEST_TYPE_SLOT_MAP.get(test_type, test_type)
|
||||
if template_config_id:
|
||||
template_config = await session.get(TemplateConfig, template_config_id)
|
||||
if template_config:
|
||||
slot_result = await session.exec(
|
||||
select(TemplateSlot).where(
|
||||
TemplateSlot.config_id == template_config.id,
|
||||
TemplateSlot.slot_name == slot_name,
|
||||
)
|
||||
)
|
||||
locale_map: dict[str, str] = {}
|
||||
for s in slot_result.all():
|
||||
locale_map[s.locale] = s.template
|
||||
if locale_map:
|
||||
template_slots = {EventType.SCHEDULED_MESSAGE.value: locale_map}
|
||||
|
||||
# Resolve target config + receivers (same as watcher — this already sets
|
||||
# each receiver.locale from TargetReceiver.locale or TelegramChat override)
|
||||
resolved = await _resolve_target(session, target)
|
||||
|
||||
target_cfg = TargetConfig(
|
||||
type=resolved["target_type"],
|
||||
config=resolved["target_config"],
|
||||
template_slots=template_slots,
|
||||
locale=locale,
|
||||
date_format=template_config.date_format if template_config else "%d.%m.%Y, %H:%M UTC",
|
||||
date_only_format=template_config.date_only_format if template_config and template_config.date_only_format else "%d.%m.%Y",
|
||||
provider_api_key=provider_config.get("api_key"),
|
||||
provider_internal_url=provider_config.get("url", ""),
|
||||
provider_external_url=provider_config.get("external_domain", ""),
|
||||
receivers=resolved["receivers"],
|
||||
)
|
||||
|
||||
if not template_slots:
|
||||
if not template_config_id:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
"This tracker has no Template Config linked (neither on the "
|
||||
"tracker's default nor on this target link). Assign one in the "
|
||||
"tracker settings and make sure it defines a "
|
||||
f"'{slot_name}' slot."
|
||||
),
|
||||
}
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
f"No '{slot_name}' template defined in the linked Template Config "
|
||||
f"'{template_config.name if template_config else template_config_id}' "
|
||||
f"(locale: {locale}). Add the slot under Template Configs."
|
||||
),
|
||||
}
|
||||
|
||||
# Fetch assets and build event
|
||||
try:
|
||||
event = await _build_event(
|
||||
provider_type=provider.type,
|
||||
provider_config=provider_config,
|
||||
provider_name=provider.name or provider.type,
|
||||
tracker_name=tracker.name or "",
|
||||
tracker_filters=dict(tracker.filters) if tracker.filters else {},
|
||||
collection_ids=collection_ids,
|
||||
test_type=test_type,
|
||||
tracking_config=tracking_config,
|
||||
)
|
||||
except Exception as err: # noqa: BLE001
|
||||
_LOGGER.exception("Test dispatch event build failed")
|
||||
return {"success": False, "error": f"Provider connection failed: {err}"}
|
||||
if event is None:
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
"Provider returned no data. Check that the provider is reachable, "
|
||||
"credentials are valid, and the tracker has collections configured."
|
||||
),
|
||||
}
|
||||
# Periodic summary only needs album stats (extra.albums), not assets — skip the asset check.
|
||||
if not event.added_assets and test_type in ("scheduled", "memory"):
|
||||
return {
|
||||
"success": False,
|
||||
"error": (
|
||||
"No matching assets found. Verify the tracker's albums contain assets "
|
||||
"that pass the tracking config filters (favorites only, rating, asset type)."
|
||||
) + (" for today" if test_type == "memory" else ""),
|
||||
}
|
||||
|
||||
# Dispatch through the real NotificationDispatcher
|
||||
url_cache, asset_cache = await _get_telegram_caches()
|
||||
dispatcher = NotificationDispatcher(url_cache=url_cache, asset_cache=asset_cache)
|
||||
results = await dispatcher.dispatch(event, [target_cfg])
|
||||
|
||||
if not results:
|
||||
return {"success": False, "error": "No dispatch results"}
|
||||
return results[0]
|
||||
|
||||
|
||||
async def _build_event(
|
||||
*,
|
||||
provider_type: str,
|
||||
provider_config: dict,
|
||||
provider_name: str,
|
||||
tracker_name: str,
|
||||
tracker_filters: dict,
|
||||
collection_ids: list[str],
|
||||
test_type: str,
|
||||
tracking_config: TrackingConfig | None = None,
|
||||
) -> ServiceEvent | None:
|
||||
"""Build a ServiceEvent with real provider data."""
|
||||
from datetime import datetime, timezone
|
||||
|
||||
if provider_type == "immich":
|
||||
if test_type == "periodic":
|
||||
return await _build_immich_periodic_event(
|
||||
provider_config=provider_config,
|
||||
provider_name=provider_name,
|
||||
tracker_name=tracker_name,
|
||||
collection_ids=collection_ids,
|
||||
)
|
||||
return await _build_immich_event(
|
||||
provider_config=provider_config,
|
||||
provider_name=provider_name,
|
||||
tracker_name=tracker_name,
|
||||
collection_ids=collection_ids,
|
||||
test_type=test_type,
|
||||
tracking_config=tracking_config,
|
||||
)
|
||||
elif provider_type == "scheduler":
|
||||
from notify_bridge_core.providers.scheduler import SchedulerServiceProvider
|
||||
custom_vars = tracker_filters.get("custom_variables", {})
|
||||
sched = SchedulerServiceProvider(
|
||||
name=provider_name,
|
||||
tracker_name=tracker_name,
|
||||
custom_variables=custom_vars,
|
||||
)
|
||||
events, _ = await sched.poll(collection_ids, {})
|
||||
return events[0] if events else None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
async def _build_immich_event(
|
||||
*,
|
||||
provider_config: dict,
|
||||
provider_name: str,
|
||||
tracker_name: str,
|
||||
collection_ids: list[str],
|
||||
test_type: str,
|
||||
tracking_config: TrackingConfig | None = None,
|
||||
) -> ServiceEvent | None:
|
||||
"""Build an Immich scheduled/memory event using shared core utilities."""
|
||||
from datetime import datetime, timezone
|
||||
from notify_bridge_core.providers.immich import ImmichServiceProvider
|
||||
from notify_bridge_core.providers.immich.asset_utils import collect_scheduled_assets
|
||||
from notify_bridge_core.providers.immich.models import ImmichAlbumData, SharedLinkInfo
|
||||
|
||||
ext_domain = provider_config.get("external_domain") or provider_config.get("url", "")
|
||||
prefix = "memory" if test_type == "memory" else "scheduled"
|
||||
limit = getattr(tracking_config, f"{prefix}_limit", 10) if tracking_config else 10
|
||||
asset_type = getattr(tracking_config, f"{prefix}_asset_type", "all") if tracking_config else "all"
|
||||
favorite_only = getattr(tracking_config, f"{prefix}_favorite_only", False) if tracking_config else False
|
||||
min_rating = getattr(tracking_config, f"{prefix}_min_rating", 0) if tracking_config else 0
|
||||
memory_source = getattr(tracking_config, "memory_source", "albums") if tracking_config else "albums"
|
||||
is_memory = test_type == "memory"
|
||||
|
||||
from .http_session import get_http_session
|
||||
http_session = await get_http_session()
|
||||
immich = ImmichServiceProvider(
|
||||
http_session,
|
||||
provider_config.get("url", ""),
|
||||
provider_config.get("api_key", ""),
|
||||
provider_config.get("external_domain"),
|
||||
provider_name,
|
||||
)
|
||||
if not await immich.connect():
|
||||
return None
|
||||
|
||||
# Native Immich memories API path
|
||||
if is_memory and memory_source == "native":
|
||||
return await _build_native_memory_event(
|
||||
immich, ext_domain, provider_name, tracker_name,
|
||||
collection_ids, limit, asset_type, favorite_only, min_rating,
|
||||
)
|
||||
|
||||
# Album-based path: use shared collect_scheduled_assets.
|
||||
# Fetch albums + shared links in parallel — on a 20-album tracker the old
|
||||
# serial ``await`` loop took ~2 × 20 × RTT, now it's one round-trip.
|
||||
import asyncio as _asyncio
|
||||
album_tasks = [immich.client.get_album(aid) for aid in collection_ids]
|
||||
link_tasks = [immich.client.get_shared_links(aid) for aid in collection_ids]
|
||||
album_results, link_results = await _asyncio.gather(
|
||||
_asyncio.gather(*album_tasks, return_exceptions=True),
|
||||
_asyncio.gather(*link_tasks, return_exceptions=True),
|
||||
)
|
||||
albums: dict[str, ImmichAlbumData] = {}
|
||||
shared_links: dict[str, list[SharedLinkInfo]] = {}
|
||||
for album_id, album, links in zip(collection_ids, album_results, link_results):
|
||||
if isinstance(album, Exception) or album is None:
|
||||
continue
|
||||
albums[album_id] = album
|
||||
shared_links[album_id] = links if not isinstance(links, Exception) else []
|
||||
|
||||
assets, collections_extra = collect_scheduled_assets(
|
||||
albums, shared_links, ext_domain,
|
||||
limit=limit,
|
||||
asset_type=asset_type,
|
||||
favorite_only=favorite_only,
|
||||
min_rating=min_rating,
|
||||
is_memory=is_memory,
|
||||
)
|
||||
|
||||
first_col = collections_extra[0] if collections_extra else {}
|
||||
return ServiceEvent(
|
||||
event_type=EventType.SCHEDULED_MESSAGE,
|
||||
provider_type=ServiceProviderType.IMMICH,
|
||||
provider_name=provider_name,
|
||||
collection_id=collection_ids[0] if collection_ids else "",
|
||||
collection_name=first_col.get("name", tracker_name),
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
added_assets=assets,
|
||||
added_count=len(assets),
|
||||
extra={
|
||||
"collections": collections_extra,
|
||||
"albums": collections_extra,
|
||||
**(first_col if first_col else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _build_immich_periodic_event(
|
||||
*,
|
||||
provider_config: dict,
|
||||
provider_name: str,
|
||||
tracker_name: str,
|
||||
collection_ids: list[str],
|
||||
) -> ServiceEvent | None:
|
||||
"""Build a periodic-summary event (album stats only, no assets).
|
||||
|
||||
Reuses the same shared core utility (`collect_scheduled_assets`) that
|
||||
scheduled/memory tests use, invoked with limit=0 so we get the full
|
||||
``collections_extra`` block (album name/url/counts/...) without selecting
|
||||
any individual assets — which is exactly what the
|
||||
``periodic_summary_message`` template renders.
|
||||
"""
|
||||
from datetime import datetime, timezone
|
||||
from notify_bridge_core.providers.immich import ImmichServiceProvider
|
||||
from notify_bridge_core.providers.immich.asset_utils import collect_scheduled_assets
|
||||
from notify_bridge_core.providers.immich.models import ImmichAlbumData, SharedLinkInfo
|
||||
|
||||
from .http_session import get_http_session
|
||||
http_session = await get_http_session()
|
||||
immich = ImmichServiceProvider(
|
||||
http_session,
|
||||
provider_config.get("url", ""),
|
||||
provider_config.get("api_key", ""),
|
||||
provider_config.get("external_domain"),
|
||||
provider_name,
|
||||
)
|
||||
if not await immich.connect():
|
||||
return None
|
||||
|
||||
ext_domain = provider_config.get("external_domain") or provider_config.get("url", "")
|
||||
|
||||
# Parallel fetch — see _build_immich_event above for the same rationale.
|
||||
import asyncio as _asyncio
|
||||
album_tasks = [immich.client.get_album(aid) for aid in collection_ids]
|
||||
link_tasks = [immich.client.get_shared_links(aid) for aid in collection_ids]
|
||||
album_results, link_results = await _asyncio.gather(
|
||||
_asyncio.gather(*album_tasks, return_exceptions=True),
|
||||
_asyncio.gather(*link_tasks, return_exceptions=True),
|
||||
)
|
||||
albums: dict[str, ImmichAlbumData] = {}
|
||||
shared_links: dict[str, list[SharedLinkInfo]] = {}
|
||||
for album_id, album, links in zip(collection_ids, album_results, link_results):
|
||||
if isinstance(album, Exception) or album is None:
|
||||
continue
|
||||
albums[album_id] = album
|
||||
shared_links[album_id] = links if not isinstance(links, Exception) else []
|
||||
|
||||
# limit=0 → returns ([], collections_extra) with full per-album stats.
|
||||
_assets, collections_extra = collect_scheduled_assets(
|
||||
albums, shared_links, ext_domain,
|
||||
limit=0,
|
||||
asset_type="all",
|
||||
favorite_only=False,
|
||||
min_rating=0,
|
||||
is_memory=False,
|
||||
)
|
||||
|
||||
first_col = collections_extra[0] if collections_extra else {}
|
||||
return ServiceEvent(
|
||||
event_type=EventType.SCHEDULED_MESSAGE,
|
||||
provider_type=ServiceProviderType.IMMICH,
|
||||
provider_name=provider_name,
|
||||
collection_id=collection_ids[0] if collection_ids else "",
|
||||
collection_name=first_col.get("name", tracker_name),
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
added_assets=[],
|
||||
added_count=0,
|
||||
extra={
|
||||
"collections": collections_extra,
|
||||
"albums": collections_extra,
|
||||
**(first_col if first_col else {}),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _build_native_memory_event(
|
||||
immich,
|
||||
ext_domain: str,
|
||||
provider_name: str,
|
||||
tracker_name: str,
|
||||
collection_ids: list[str],
|
||||
limit: int,
|
||||
asset_type: str,
|
||||
favorite_only: bool,
|
||||
min_rating: int,
|
||||
) -> ServiceEvent | None:
|
||||
"""Build event from Immich native memories API."""
|
||||
import random
|
||||
from datetime import datetime, timezone
|
||||
from notify_bridge_core.models.media import MediaAsset, MediaType
|
||||
from notify_bridge_core.providers.immich.asset_utils import filter_assets
|
||||
from notify_bridge_core.providers.immich.models import ImmichAssetInfo
|
||||
|
||||
memories = await immich.client.get_memories()
|
||||
tracked_ids = set(collection_ids) if collection_ids else None
|
||||
|
||||
# Collect raw assets, convert to ImmichAssetInfo for unified filtering
|
||||
raw_assets: list[ImmichAssetInfo] = []
|
||||
year_map: dict[str, int | None] = {} # asset_id → memory year
|
||||
for mem in memories:
|
||||
mem_year = mem.get("data", {}).get("year")
|
||||
for raw in mem.get("assets", []):
|
||||
asset_id = raw.get("id", "")
|
||||
if tracked_ids:
|
||||
asset_albums = raw.get("albums", [])
|
||||
if not any(a.get("id") in tracked_ids for a in asset_albums):
|
||||
continue
|
||||
asset = ImmichAssetInfo.from_api_response(raw)
|
||||
if not asset.is_processed:
|
||||
continue
|
||||
raw_assets.append(asset)
|
||||
year_map[asset_id] = mem_year
|
||||
|
||||
# Apply standard filters (no memory_date — native API already filters by date)
|
||||
filtered = filter_assets(
|
||||
raw_assets,
|
||||
favorite_only=favorite_only,
|
||||
min_rating=min_rating,
|
||||
asset_type=asset_type,
|
||||
)
|
||||
|
||||
# Random sample
|
||||
if len(filtered) > limit:
|
||||
selected = random.sample(filtered, limit)
|
||||
else:
|
||||
random.shuffle(filtered)
|
||||
selected = filtered
|
||||
|
||||
from notify_bridge_core.providers.immich.asset_utils import asset_to_media
|
||||
|
||||
all_assets = []
|
||||
for asset in selected:
|
||||
media = asset_to_media(asset, ext_domain)
|
||||
media.extra["year"] = year_map.get(asset.id)
|
||||
all_assets.append(media)
|
||||
|
||||
return ServiceEvent(
|
||||
event_type=EventType.SCHEDULED_MESSAGE,
|
||||
provider_type=ServiceProviderType.IMMICH,
|
||||
provider_name=provider_name,
|
||||
collection_id=collection_ids[0] if collection_ids else "",
|
||||
collection_name=tracker_name,
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
added_assets=all_assets,
|
||||
added_count=len(all_assets),
|
||||
extra={
|
||||
"collections": [],
|
||||
"albums": [],
|
||||
},
|
||||
)
|
||||
Reference in New Issue
Block a user