feat: deferred dispatch, release-check provider, settings polish

- Defer quiet-hours dispatches into new deferred_dispatch table; drain job + periodic catch-up scan re-fire at window end with coalescing on (link, event_type, collection_id). - Add ON DELETE SET NULL migration on event_log_id and partial unique index on (link_id, collection_id, event_type) WHERE status='pending'. - Add release-check provider abstraction (Gitea/GitHub) with SSRF-safe URL validation, settings UI cassette, and scheduled polling. - Replace importlib-only version lookup with version.py helper that prefers the higher of installed metadata vs source pyproject so stale editable dev installs stop misreporting. - Aurora frontend polish: MetaStrip component, ReleaseCassette, EventDetailModal expansion, and i18n additions.
2026-05-12 02:58:07 +03:00
parent bb5afcc222
commit ba199f24bd
47 changed files with 5627 additions and 290 deletions
@@ -2,13 +2,18 @@

 import logging
 import os
+from urllib.parse import urlparse

-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, HTTPException, Request
 from pydantic import BaseModel
 from sqlmodel import select
 from sqlmodel.ext.asyncio.session import AsyncSession

+from notify_bridge_core.notifications.ssrf import UnsafeURLError, avalidate_outbound_url
+from notify_bridge_core.release import ReleaseProviderKind, is_valid_repo
+
 from ..auth.dependencies import get_current_user, require_admin
+from ..auth.routes import limiter  # shared SlowAPI instance (app.state.limiter)
 from ..database.engine import get_session
 from ..database.models import AppSetting, TelegramBot, User

@@ -28,6 +33,12 @@ _SETTING_KEYS = {
    "log_level": "NOTIFY_BRIDGE_LOG_LEVEL",  # DEBUG/INFO/WARNING/ERROR
    "log_format": "NOTIFY_BRIDGE_LOG_FORMAT",  # text|json (requires restart to switch)
    "log_levels": "NOTIFY_BRIDGE_LOG_LEVELS",  # module=LEVEL,module2=LEVEL
+    # Release-check — see services/release_check.py for the cached-state keys.
+    "release_provider_kind": "NOTIFY_BRIDGE_RELEASE_PROVIDER",  # disabled|gitea|github
+    "release_provider_url": "NOTIFY_BRIDGE_RELEASE_PROVIDER_URL",
+    "release_provider_repo": "NOTIFY_BRIDGE_RELEASE_PROVIDER_REPO",
+    "release_include_prereleases": None,  # "0"|"1"
+    "release_check_interval_hours": None,  # 1..168
 }

 _DEFAULTS = {
@@ -42,6 +53,13 @@ _DEFAULTS = {
    "log_level": "INFO",
    "log_format": "text",
    "log_levels": "",
+    # Pre-seed Gitea release checks against this repo's own upstream so a fresh
+    # install knows where to look without operator intervention.
+    "release_provider_kind": "gitea",
+    "release_provider_url": "https://git.dolgolyov-family.by",
+    "release_provider_repo": "alexei.dolgolyov/notify-bridge",
+    "release_include_prereleases": "0",
+    "release_check_interval_hours": "12",
 }

 # Settings whose changes require dropping in-memory Telegram caches so the
@@ -53,6 +71,17 @@ _CACHE_SETTING_KEYS = {"telegram_cache_ttl_hours", "telegram_asset_cache_max_ent
 # changing it means swapping the handler formatter entirely.
 _LOG_SETTING_KEYS = {"log_level", "log_levels", "log_format"}

+# Release-check settings whose change must trigger cache invalidation (so a
+# stale "latest version" doesn't linger after pointing at a new repo) and a
+# scheduler re-arm so the new interval/provider takes effect immediately.
+_RELEASE_PROVIDER_KEYS = {
+    "release_provider_kind",
+    "release_provider_url",
+    "release_provider_repo",
+    "release_include_prereleases",
+}
+_RELEASE_INTERVAL_KEY = "release_check_interval_hours"
+

 async def get_setting(session: AsyncSession, key: str) -> str:
    """Read a setting from DB, falling back to env var then default."""
@@ -81,6 +110,11 @@ class SettingsUpdate(BaseModel):
    log_level: str | None = None
    log_format: str | None = None
    log_levels: str | None = None
+    release_provider_kind: str | None = None
+    release_provider_url: str | None = None
+    release_provider_repo: str | None = None
+    release_include_prereleases: bool | int | str | None = None
+    release_check_interval_hours: int | str | None = None


@router.get("")
@@ -111,12 +145,65 @@ async def update_settings(
    old_cache_values = {k: await get_setting(session, k) for k in _CACHE_SETTING_KEYS}
    old_timezone = await get_setting(session, "timezone")
    old_log_values = {k: await get_setting(session, k) for k in _LOG_SETTING_KEYS}
+    old_release_values = {k: await get_setting(session, k) for k in _RELEASE_PROVIDER_KEYS}
+    old_release_interval = await get_setting(session, _RELEASE_INTERVAL_KEY)

    for key in _SETTING_KEYS:
        value = getattr(body, key, None)
        if value is None:
            continue
-        value_str = str(value)
+        # Normalise per-key before storing so the cache keys always hold the
+        # canonical wire format ("0"/"1" for bool flags, clamped int for the
+        # release interval). Without this, str(True) would leak "True" into the
+        # release_include_prereleases cell and silently disable filtering.
+        if key == "release_include_prereleases":
+            if isinstance(value, bool):
+                value_str = "1" if value else "0"
+            else:
+                value_str = "1" if str(value).strip().lower() in ("1", "true", "yes", "on") else "0"
+        elif key == "release_check_interval_hours":
+            from ..services.release_check import parse_interval_hours
+            value_str = str(parse_interval_hours(str(value)))
+        elif key == "release_provider_kind":
+            # Reject anything outside the enum so a typo doesn't leave the DB
+            # in a state the service can't interpret.
+            value_str = str(value).strip().lower()
+            try:
+                value_str = ReleaseProviderKind(value_str).value
+            except ValueError as err:
+                raise HTTPException(
+                    status_code=400,
+                    detail=f"Invalid release_provider_kind: {value_str!r}",
+                ) from err
+        elif key == "release_provider_url":
+            value_str = str(value).strip()
+            if value_str:
+                # Reject embedded userinfo (http://user:pass@host) so the
+                # GET /settings response can never echo credentials back, and
+                # block private/loopback/metadata targets via the SSRF guard.
+                parsed = urlparse(value_str)
+                if parsed.username or parsed.password:
+                    raise HTTPException(
+                        status_code=400,
+                        detail="release_provider_url must not contain credentials",
+                    )
+                try:
+                    await avalidate_outbound_url(value_str)
+                except UnsafeURLError as err:
+                    raise HTTPException(
+                        status_code=400,
+                        detail=f"Invalid release_provider_url: {err}",
+                    ) from err
+        elif key == "release_provider_repo":
+            value_str = str(value).strip()
+            if value_str and not is_valid_repo(value_str):
+                raise HTTPException(
+                    status_code=400,
+                    detail="release_provider_repo must match 'owner/name' "
+                           "(alphanumerics, dot, dash, underscore only)",
+                )
+        else:
+            value_str = str(value)
        # GET masks the webhook secret as "***<last4>" so the real value is
        # never exposed to the frontend. If the client sends the mask back
        # (which happens on every save, since bind:value holds whatever GET
@@ -182,6 +269,27 @@ async def update_settings(
    if new_base_url and (new_base_url != old_base_url or new_secret != old_secret):
        await _reregister_webhooks(session, new_base_url, new_secret)

+    # Release-check: clear stale cache when the provider repo/url/kind changes,
+    # and re-arm the periodic job whenever the interval or provider changes.
+    new_release_values = {k: await get_setting(session, k) for k in _RELEASE_PROVIDER_KEYS}
+    new_release_interval = await get_setting(session, _RELEASE_INTERVAL_KEY)
+    release_provider_changed = new_release_values != old_release_values
+    release_interval_changed = new_release_interval != old_release_interval
+    if release_provider_changed:
+        from datetime import datetime, timezone
+
+        from notify_bridge_core.release import ReleaseErrorCode
+
+        from ..services.release_check import persist_release_state
+        await persist_release_state(
+            checked_at=datetime.now(timezone.utc).isoformat(),
+            error=ReleaseErrorCode.PROVIDER_CHANGED.value,
+            info=None,
+        )
+    if release_provider_changed or release_interval_changed:
+        from ..services.scheduler import reschedule_release_check
+        await reschedule_release_check()
+
    result = {}
    for key in _SETTING_KEYS:
        result[key] = await get_setting(session, key)
@@ -231,6 +339,122 @@ async def get_external_url(
    return {"external_url": (await get_setting(session, "external_url")).rstrip("/")}


+def _status_payload(status, *, is_admin: bool) -> dict:
+    """Serialise a :class:`ReleaseStatus` for the API.
+
+    Non-admin payloads strip the upstream release body (an XSS landmine —
+    arbitrary attacker-controlled markdown should never reach a non-admin
+    UI unless we explicitly sanitise it for display) and replace the raw
+    error string with a coarse ``error`` / ``ok`` marker so internal
+    hostnames from probe failures can't leak via the badge.
+    """
+    payload = {
+        "provider": status.provider,
+        "current": status.current,
+        "latest": status.latest,
+        "latest_tag": status.latest_tag,
+        "latest_url": status.latest_url,
+        "latest_name": status.latest_name,
+        "latest_published_at": status.latest_published_at,
+        "latest_prerelease": status.latest_prerelease,
+        "checked_at": status.checked_at,
+        "update_available": status.update_available,
+    }
+    if is_admin:
+        payload["latest_body"] = status.latest_body
+        payload["error"] = status.error
+    else:
+        payload["latest_body"] = None
+        payload["error"] = None if not status.error else "error"
+    return payload
+
+
+@router.get("/release")
+async def get_release_status(
+    user: User = Depends(get_current_user),
+):
+    """Return the cached upstream release status (no network call).
+
+    Available to all authenticated users so the sidebar badge can render for
+    everyone — admins manage the configuration but the awareness is global.
+    """
+    from ..services.release_check import load_status
+    return _status_payload(await load_status(), is_admin=(user.role == "admin"))
+
+
+@router.post("/release/check")
+@limiter.limit("6/minute")
+async def force_release_check(
+    request: Request,
+    user: User = Depends(require_admin),
+):
+    """Force an immediate upstream check and return the refreshed status."""
+    from ..services.release_check import run_check
+    status = await run_check(force=True)
+    return _status_payload(status, is_admin=True)
+
+
+class ReleaseTestRequest(BaseModel):
+    provider_kind: str
+    provider_url: str | None = None
+    provider_repo: str | None = None
+    include_prereleases: bool | None = False
+
+
+@router.post("/release/test")
+@limiter.limit("12/minute")
+async def test_release_provider(
+    request: Request,
+    body: ReleaseTestRequest,
+    user: User = Depends(require_admin),
+):
+    """Dry-run an arbitrary provider config — used by the cassette's Test button.
+
+    Validates the provider URL on the spot (SSRF + userinfo) so the operator
+    sees an actionable error before any outbound request fires.
+    """
+    from notify_bridge_core.release import ReleaseErrorCode, build_release_provider
+
+    from ..services.http_session import get_http_session
+
+    test_url = (body.provider_url or "").strip()
+    test_repo = (body.provider_repo or "").strip()
+
+    if test_repo and not is_valid_repo(test_repo):
+        return {"ok": False, "info": None, "error": ReleaseErrorCode.MISCONFIGURED.value}
+    if test_url:
+        parsed = urlparse(test_url)
+        if parsed.username or parsed.password:
+            return {"ok": False, "info": None, "error": ReleaseErrorCode.UNSAFE_URL.value}
+        try:
+            await avalidate_outbound_url(test_url)
+        except UnsafeURLError:
+            return {"ok": False, "info": None, "error": ReleaseErrorCode.UNSAFE_URL.value}
+
+    http = await get_http_session()
+    provider = build_release_provider(
+        body.provider_kind,
+        session=http,
+        url=test_url,
+        repo=test_repo,
+    )
+    if provider is None:
+        return {"ok": False, "info": None, "error": ReleaseErrorCode.MISCONFIGURED.value}
+    result = await provider.test()
+    info = result.get("info")
+    info_dict = None
+    if info is not None:
+        info_dict = {
+            "tag": info.tag,
+            "version": info.version,
+            "name": info.name,
+            "url": info.url,
+            "published_at": info.published_at,
+            "prerelease": info.prerelease,
+        }
+    return {"ok": result["ok"], "info": info_dict, "error": result.get("error")}
+
+
 async def _reregister_webhooks(
    session: AsyncSession, base_url: str, secret: str
 ) -> None:
@@ -28,8 +28,9 @@ from ..database.models import (
    WebhookPayloadLog,
 )
 from ..services.dispatch_helpers import (
+    GateReason,
    apply_tracking_display_filters,
-    event_allowed_by_config,
+    evaluate_event_gate,
    get_app_timezone,
    load_link_data,
 )
@@ -164,7 +165,16 @@ async def _dispatch_webhook_event(
        Number of successfully dispatched notifications.
    """
    dispatched = 0
+    # ``defers_to_schedule`` is collected during the loop and flushed AFTER the
+    # main session commits — the only side-effect of failing to schedule is a
+    # delayed delivery (the startup loader / catch-up scan will reschedule),
+    # so this is best-effort and must not roll back the DB writes.
+    defers_to_schedule: set[Any] = set()
    async with AsyncSession(engine) as session:
+        # App timezone is identical across trackers within one webhook request;
+        # pull it once.
+        app_tz = await get_app_timezone(session)
+
        tracker_result = await session.exec(
            select(NotificationTracker).where(
                NotificationTracker.provider_id == provider_id,
@@ -173,6 +183,8 @@ async def _dispatch_webhook_event(
        )
        trackers = tracker_result.all()

+        from ..services.deferred_dispatch import defer_event, is_deferrable
+
        for tracker in trackers:
            filters = tracker.filters or {}
            if not _passes_filters(event, filters):
@@ -185,11 +197,9 @@ async def _dispatch_webhook_event(
            if not link_data:
                continue

-            app_tz = await get_app_timezone(session)
-
            # Log event
            extra_details = {k: v for k, v in event.extra.items() if k in detail_keys}
-            session.add(EventLog(
+            event_log_row = EventLog(
                user_id=tracker.user_id,
                tracker_id=tracker.id,
                tracker_name=tracker.name,
@@ -203,18 +213,90 @@ async def _dispatch_webhook_event(
                    "provider_type": event.provider_type.value,
                    **extra_details,
                },
-            ))
+            )
+            session.add(event_log_row)
+            await session.flush()
+            event_log_id = event_log_row.id

-            # Dispatch to targets
+            # Dedupe defers by parent ``link_id``: broadcast links emit one
+            # ``link_data`` entry per child, all sharing the same parent id —
+            # the deferred row is one-per-link, so we only call ``defer_event``
+            # once per distinct id (earliest fire_at wins on ties).
+            groups: dict[int, tuple[Any, list[TargetConfig]]] = {}
+            defers_for_event: dict[int, Any] = {}
+            for ld in link_data:
+                tc = ld["tracking_config"]
+                if tc is not None:
+                    outcome = evaluate_event_gate(event, tc, app_tz)
+                    if outcome.reason is GateReason.QUIET_HOURS:
+                        if is_deferrable(event.event_type.value) and outcome.quiet_hours_end_at is not None:
+                            link_id = ld.get("link_id")
+                            if link_id is not None:
+                                prior = defers_for_event.get(link_id)
+                                if prior is None or outcome.quiet_hours_end_at < prior:
+                                    defers_for_event[link_id] = outcome.quiet_hours_end_at
+                        continue
+                    if outcome.reason is GateReason.EVENT_TYPE_DISABLED:
+                        continue
+
+                tmpl = ld["template_config"]
+                target_cfg = TargetConfig(
+                    type=ld["target_type"],
+                    config=ld["target_config"],
+                    template_slots=ld["template_slots"],
+                    date_format=tmpl.date_format if tmpl else "%d.%m.%Y, %H:%M UTC",
+                    date_only_format=tmpl.date_only_format if tmpl and tmpl.date_only_format else "%d.%m.%Y",
+                    provider_api_key=provider_config.get("api_token"),
+                    provider_internal_url=provider_config.get("url", ""),
+                    provider_external_url=provider_config.get("url", ""),
+                    receivers=ld["receivers"],
+                )
+                key = id(tc) if tc is not None else 0
+                if key not in groups:
+                    groups[key] = (tc, [])
+                groups[key][1].append(target_cfg)
+
+            # Persist defers + stamp event_log dispatch_status in the same
+            # session that holds the EventLog row, so the "deferred" badge
+            # only appears if the underlying queue rows actually exist.
+            if defers_for_event:
+                earliest = min(defers_for_event.values())
+                for link_id, fire_at in defers_for_event.items():
+                    await defer_event(
+                        session,
+                        event=event,
+                        user_id=tracker.user_id,
+                        tracker_id=tracker.id,
+                        link_id=link_id,
+                        event_log_id=event_log_id,
+                        fire_at=fire_at,
+                    )
+                details = dict(event_log_row.details or {})
+                if not details.get("dispatch_status"):
+                    details["dispatch_status"] = "deferred"
+                    details["deferred_until"] = earliest.isoformat()
+                    event_log_row.details = details
+                    session.add(event_log_row)
+                defers_to_schedule.update(defers_for_event.values())
+
+            # Dispatch to targets. Isolate dispatcher exceptions per group so
+            # a failed remote call doesn't bubble out, abort the surrounding
+            # transaction, and roll back the just-written defers/event_log.
            from ..services.http_session import get_http_session
            dispatcher = NotificationDispatcher(session=await get_http_session())
-            for tc, target_configs in _build_target_groups(event, link_data, provider_config, app_tz):
+            for tc, target_configs in groups.values():
                if not target_configs:
                    continue
                shaped_event = apply_tracking_display_filters(event, tc)
                if shaped_event is None:
                    continue
-                results = await dispatcher.dispatch(shaped_event, target_configs)
+                try:
+                    results = await dispatcher.dispatch(shaped_event, target_configs)
+                except Exception as err:  # noqa: BLE001
+                    _LOGGER.exception(
+                        "Dispatcher raised for tracker %d: %s", tracker.id, err,
+                    )
+                    continue
                for r in results:
                    if r.get("success"):
                        dispatched += 1
@@ -226,6 +308,18 @@ async def _dispatch_webhook_event(

        await session.commit()

+    # Schedule drain jobs OUTSIDE the DB session so an APScheduler hiccup
+    # can't roll back the persisted defer rows.
+    if defers_to_schedule:
+        from ..services.scheduler import schedule_deferred_drain
+        for fire_at in defers_to_schedule:
+            try:
+                schedule_deferred_drain(fire_at)
+            except Exception:  # noqa: BLE001
+                _LOGGER.exception(
+                    "Failed to schedule deferred drain for %s", fire_at,
+                )
+
    return dispatched


@@ -554,41 +648,3 @@ async def generic_webhook(token: str, request: Request):
            await log_session.commit()

    return {"ok": True, "dispatched": dispatched}
-
-
-def _build_target_groups(
-    event: ServiceEvent,
-    link_data: list[dict[str, Any]],
-    provider_config: dict[str, Any],
-    app_tz: str = "UTC",
-) -> list[tuple[Any, list[TargetConfig]]]:
-    """Build TargetConfigs for dispatch, grouped by their TrackingConfig.
-
-    Targets sharing a TrackingConfig dispatch together so a single
-    ``apply_tracking_display_filters`` pass can shape one event for the
-    whole group; targets with different TCs may see differently-shaped
-    events (e.g. one with favorites_only, one without).
-    """
-    groups: dict[int, tuple[Any, list[TargetConfig]]] = {}
-    for ld in link_data:
-        tc = ld["tracking_config"]
-        if tc and not event_allowed_by_config(event, tc, app_tz):
-            continue
-
-        tmpl = ld["template_config"]
-        target_cfg = TargetConfig(
-            type=ld["target_type"],
-            config=ld["target_config"],
-            template_slots=ld["template_slots"],
-            date_format=tmpl.date_format if tmpl else "%d.%m.%Y, %H:%M UTC",
-            date_only_format=tmpl.date_only_format if tmpl and tmpl.date_only_format else "%d.%m.%Y",
-            provider_api_key=provider_config.get("api_token"),
-            provider_internal_url=provider_config.get("url", ""),
-            provider_external_url=provider_config.get("url", ""),
-            receivers=ld["receivers"],
-        )
-        key = id(tc) if tc is not None else 0
-        if key not in groups:
-            groups[key] = (tc, [])
-        groups[key][1].append(target_cfg)
-    return list(groups.values())