alexei.dolgolyov/notify-bridge
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity
Files
ab621b6abc7497a312e22554fb8cd7830d90d5a5
notify-bridge/packages/server/src/notify_bridge_server/api/webhooks.py
T
alexei.dolgolyov ab621b6abc feat: wire tracking-config display filters + per-tracker adaptive polling 
Display filters (Immich tracking config):
- favorites_only drops events with no favorited new assets, or filters
  added_assets to favorites only
- assets_order_by/assets_order sort the rendered list
  (date / name / rating / random / none)
- max_assets_to_show caps rendered+attached media (default 5 -> 10)
- include_tags strips people from event extras and tags from each asset
- include_asset_details strips city/country/state/lat/lon/is_favorite/
  rating/description; load-bearing fields (thumbhash, file_size,
  playback_size, cache keys) preserved
- New apply_tracking_display_filters helper in dispatch_helpers; wired
  into watcher, webhooks, scheduled/periodic/memory, and manual
  test-dispatch
- Targets sharing a TrackingConfig dispatch together; targets with
  different TCs each see their own shaped event

Adaptive polling:
- Replace NotificationTracker.batch_duration with adaptive_max_skip
- Per-tracker opt-in: NULL/0 disables back-off (every tick runs);
  positive N caps the skip factor at (N-1)-in-N after long idle
- Scheduler caches the cap in module state for the tick fast-path
- Migration adds the new column; API schemas/responses, frontend types,
  i18n, and the tracker form updated to match
2026-04-24 21:12:10 +03:00

595 lines
21 KiB
Python
Raw Blame History

"""Incoming webhook handlers for webhook-based providers (Gitea, etc.)."""
from __future__ import annotations
import hashlib
import hmac
import json
import logging
from typing import Any
from fastapi import APIRouter, HTTPException, Request
from sqlmodel import select
from sqlmodel.ext.asyncio.session import AsyncSession
from notify_bridge_core.models.events import ServiceEvent
from notify_bridge_core.notifications.dispatcher import NotificationDispatcher, TargetConfig
from notify_bridge_core.providers.gitea.event_parser import parse_webhook as parse_gitea_webhook
from notify_bridge_core.providers.planka.event_parser import parse_webhook as parse_planka_webhook
from notify_bridge_core.providers.webhook.event_parser import parse_webhook as parse_generic_webhook
from ..database.engine import get_engine
from sqlalchemy import delete as sa_delete, func
from ..database.models import (
EventLog,
NotificationTracker,
ServiceProvider,
WebhookPayloadLog,
)
from ..services.dispatch_helpers import (
apply_tracking_display_filters,
event_allowed_by_config,
get_app_timezone,
load_link_data,
)
_LOGGER = logging.getLogger(__name__)
router = APIRouter(prefix="/api/webhooks", tags=["webhooks"])
# Hard cap on inbound webhook body size (1 MiB is far larger than anything
# legitimate providers send and keeps the worst-case memory footprint bounded
# when a malicious peer lies about Content-Length or streams slowly).
_MAX_WEBHOOK_BODY_BYTES = 1_000_000
async def _read_bounded_body(request: Request, limit: int = _MAX_WEBHOOK_BODY_BYTES) -> bytes:
"""Reject oversized inbound bodies before they exhaust memory.
First checks ``Content-Length`` (fast-path for honest peers), then
streams the body in chunks enforcing the same cap on actual bytes
received so a peer that lies about Content-Length cannot slip through.
"""
declared = request.headers.get("content-length")
if declared:
try:
if int(declared) > limit:
raise HTTPException(
status_code=413,
detail=f"Payload too large (max {limit} bytes)",
)
except ValueError:
raise HTTPException(status_code=400, detail="Invalid Content-Length")
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise HTTPException(
status_code=413,
detail=f"Payload too large (max {limit} bytes)",
)
chunks.append(chunk)
return b"".join(chunks)
async def _get_provider_by_token(
session: AsyncSession, token: str, expected_type: str,
) -> ServiceProvider:
"""Look up a provider by its webhook_token and expected type."""
result = await session.exec(
select(ServiceProvider).where(
ServiceProvider.webhook_token == token,
ServiceProvider.type == expected_type,
)
)
provider = result.first()
if not provider:
raise HTTPException(status_code=404, detail="Provider not found")
return provider
# ---------------------------------------------------------------------------
# HMAC-SHA256 validation
# ---------------------------------------------------------------------------
def _verify_gitea_signature(secret: str, body: bytes, signature: str) -> bool:
"""Verify Gitea X-Gitea-Signature HMAC-SHA256."""
expected = hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
return hmac.compare_digest(expected, signature)
# ---------------------------------------------------------------------------
# Filter helpers
# ---------------------------------------------------------------------------
def _passes_filters(
event: ServiceEvent,
filters: dict[str, Any],
) -> bool:
"""Check if an event passes the tracker's filters."""
# Collection filter (repo full_name for Gitea)
collections = filters.get("collections", [])
if collections and event.collection_id not in collections:
return False
sender = event.extra.get("sender", "")
# Sender allowlist
senders = filters.get("senders", [])
if senders and sender not in senders:
return False
# Sender blocklist
exclude_senders = filters.get("exclude_senders", [])
if exclude_senders and sender in exclude_senders:
return False
return True
# ---------------------------------------------------------------------------
# Shared dispatch helper
# ---------------------------------------------------------------------------
async def _dispatch_webhook_event(
engine: Any,
provider_id: int,
provider_name: str,
provider_config: dict[str, Any],
event: ServiceEvent,
detail_keys: tuple[str, ...],
) -> int:
"""Load trackers, filter, create EventLogs, dispatch notifications, and commit.
Parameters
----------
engine:
SQLAlchemy async engine.
provider_id:
ID of the ServiceProvider that received the webhook.
provider_name:
Human-readable name of the provider (for logging).
provider_config:
The provider's ``config`` dict (passed through to target config builder).
event:
Parsed :class:`ServiceEvent` to dispatch.
detail_keys:
Keys from ``event.extra`` to include in the EventLog ``details`` dict.
Returns
-------
int
Number of successfully dispatched notifications.
"""
dispatched = 0
async with AsyncSession(engine) as session:
tracker_result = await session.exec(
select(NotificationTracker).where(
NotificationTracker.provider_id == provider_id,
NotificationTracker.enabled == True, # noqa: E712
)
)
trackers = tracker_result.all()
for tracker in trackers:
filters = tracker.filters or {}
if not _passes_filters(event, filters):
_LOGGER.debug(
"Event filtered out for tracker %d (%s)", tracker.id, tracker.name
)
continue
link_data = await load_link_data(session, tracker.id)
if not link_data:
continue
app_tz = await get_app_timezone(session)
# Log event
extra_details = {k: v for k, v in event.extra.items() if k in detail_keys}
session.add(EventLog(
user_id=tracker.user_id,
tracker_id=tracker.id,
tracker_name=tracker.name,
provider_id=provider_id,
provider_name=provider_name,
event_type=event.event_type.value,
collection_id=event.collection_id,
collection_name=event.collection_name,
assets_count=0,
details={
"provider_type": event.provider_type.value,
**extra_details,
},
))
# Dispatch to targets
from ..services.http_session import get_http_session
dispatcher = NotificationDispatcher(session=await get_http_session())
for tc, target_configs in _build_target_groups(event, link_data, provider_config, app_tz):
if not target_configs:
continue
shaped_event = apply_tracking_display_filters(event, tc)
if shaped_event is None:
continue
results = await dispatcher.dispatch(shaped_event, target_configs)
for r in results:
if r.get("success"):
dispatched += 1
else:
_LOGGER.error(
"Notification failed for tracker %d: %s",
tracker.id, r.get("error", "unknown"),
)
await session.commit()
return dispatched
# ---------------------------------------------------------------------------
# Gitea webhook endpoint
# ---------------------------------------------------------------------------
@router.post("/gitea/{token}")
async def gitea_webhook(token: str, request: Request):
"""Receive a Gitea webhook, parse it, filter, and dispatch notifications."""
engine = get_engine()
# --- Load provider and validate signature ---
async with AsyncSession(engine) as session:
provider = await _get_provider_by_token(session, token, "gitea")
webhook_secret = (provider.config or {}).get("webhook_secret", "")
# Read raw body for HMAC check
raw_body = await _read_bounded_body(request)
if not webhook_secret:
raise HTTPException(
status_code=403,
detail="Webhook secret not configured on this provider",
)
signature = request.headers.get("X-Gitea-Signature", "")
if not signature or not _verify_gitea_signature(webhook_secret, raw_body, signature):
raise HTTPException(status_code=403, detail="Invalid signature")
# Parse event header + payload
event_header = request.headers.get("X-Gitea-Event", "")
if not event_header:
return {"ok": True, "skipped": "no event header"}
try:
payload = json.loads(raw_body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
raise HTTPException(status_code=400, detail="Invalid JSON")
event = parse_gitea_webhook(event_header, payload, provider.name)
if event is None:
return {"ok": True, "skipped": "unmapped event"}
# --- Dispatch ---
dispatched = await _dispatch_webhook_event(
engine=engine,
provider_id=provider.id,
provider_name=provider.name,
provider_config=provider.config or {},
event=event,
detail_keys=(
"sender", "branch", "commit_count",
"issue_number", "issue_title",
"pr_number", "pr_title",
"release_tag", "release_name",
),
)
return {"ok": True, "dispatched": dispatched}
# ---------------------------------------------------------------------------
# Planka webhook endpoint
# ---------------------------------------------------------------------------
def _verify_planka_token(expected_token: str, request: Request) -> bool:
"""Verify Planka webhook Bearer token."""
auth_header = request.headers.get("Authorization", "")
if auth_header.startswith("Bearer "):
token = auth_header[7:]
return hmac.compare_digest(token, expected_token)
return False
@router.post("/planka/{token}")
async def planka_webhook(token: str, request: Request):
"""Receive a Planka webhook, parse it, filter, and dispatch notifications."""
engine = get_engine()
# --- Load provider and validate token ---
async with AsyncSession(engine) as session:
provider = await _get_provider_by_token(session, token, "planka")
webhook_secret = (provider.config or {}).get("webhook_secret", "")
if not webhook_secret:
raise HTTPException(
status_code=403,
detail="Webhook secret not configured on this provider",
)
if not _verify_planka_token(webhook_secret, request):
raise HTTPException(status_code=403, detail="Invalid token")
# Parse payload from the bounded raw_body we already read.
try:
payload = json.loads(raw_body.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
raise HTTPException(status_code=400, detail="Invalid JSON")
event_type = payload.get("type", "")
if not event_type:
return {"ok": True, "skipped": "no event type"}
base_url = (provider.config or {}).get("url", "")
event = parse_planka_webhook(event_type, payload, provider.name, base_url=base_url)
if event is None:
return {"ok": True, "skipped": "unmapped event"}
# --- Dispatch ---
dispatched = await _dispatch_webhook_event(
engine=engine,
provider_id=provider.id,
provider_name=provider.name,
provider_config=provider.config or {},
event=event,
detail_keys=(
"sender", "card_name", "board_name",
"list_name", "old_list_name", "new_list_name",
"comment_text", "task_name", "attachment_name",
"label_name",
),
)
return {"ok": True, "dispatched": dispatched}
# ---------------------------------------------------------------------------
# Generic Webhook endpoint
# ---------------------------------------------------------------------------
def _verify_generic_webhook_auth(
config: dict[str, Any],
request: Request,
raw_body: bytes,
) -> bool:
"""Verify authentication for a generic webhook based on configured auth_mode."""
auth_mode = config.get("auth_mode", "none")
if auth_mode == "none":
return True
secret = config.get("webhook_secret", "")
if not secret:
return False
if auth_mode == "hmac_sha256":
# Support common signature headers
signature = (
request.headers.get("X-Hub-Signature-256", "")
or request.headers.get("X-Webhook-Signature", "")
or request.headers.get("X-Signature-256", "")
)
# Strip "sha256=" prefix if present (GitHub-style)
if signature.startswith("sha256="):
signature = signature[7:]
if not signature:
return False
expected = hmac.new(secret.encode(), raw_body, hashlib.sha256).hexdigest()
return hmac.compare_digest(expected, signature)
if auth_mode == "bearer_token":
auth_header = request.headers.get("Authorization", "")
if auth_header.startswith("Bearer "):
token = auth_header[7:]
return hmac.compare_digest(token, secret)
return False
return False
_SENSITIVE_HEADER_SUBSTR = (
"token", "auth", "key", "secret", "signature", "password", "credential",
"cookie", "x-api", "x-hub-signature",
)
def _is_sensitive_header(name: str) -> bool:
n = name.lower()
return any(s in n for s in _SENSITIVE_HEADER_SUBSTR)
def _filter_headers(raw_headers: dict[str, str]) -> dict[str, str]:
"""Keep only safe headers for logging (strip Authorization, signatures, tokens).
Allowlist base set of known-safe headers, accept X-* only if they do not
match any sensitive substring (token/auth/key/secret/signature/...).
"""
safe: dict[str, str] = {}
for k, v in raw_headers.items():
kl = k.lower()
if _is_sensitive_header(kl):
continue
if kl in ("content-type", "user-agent", "content-length", "accept") or kl.startswith("x-"):
safe[k] = v
return safe
async def _save_webhook_log(
session: AsyncSession,
provider_id: int,
method: str,
headers: dict[str, str],
body: dict[str, Any] | str,
status: str,
extracted_fields: dict[str, Any] | None = None,
error_message: str = "",
max_count: int = 20,
) -> None:
"""Insert a webhook payload log entry and prune old ones."""
try:
body_json = body if isinstance(body, dict) else {}
session.add(WebhookPayloadLog(
provider_id=provider_id,
method=method,
headers=headers,
body=body_json,
status=status,
extracted_fields=extracted_fields or {},
error_message=error_message,
))
await session.flush()
# Atomic prune: DELETE anything for this provider outside the newest
# max_count rows. Avoids the COUNT -> SELECT -> DELETE race.
keep_subq = (
select(WebhookPayloadLog.id)
.where(WebhookPayloadLog.provider_id == provider_id)
.order_by(WebhookPayloadLog.created_at.desc(), WebhookPayloadLog.id.desc())
.limit(max_count)
.subquery()
)
await session.execute(
sa_delete(WebhookPayloadLog)
.where(WebhookPayloadLog.provider_id == provider_id)
.where(~WebhookPayloadLog.id.in_(select(keep_subq.c.id)))
)
except Exception:
_LOGGER.warning("Failed to save webhook payload log for provider %d", provider_id, exc_info=True)
try:
await session.rollback()
except Exception:
pass
@router.post("/webhook/{token}")
async def generic_webhook(token: str, request: Request):
"""Receive a generic webhook, extract variables via JSONPath, and dispatch notifications."""
engine = get_engine()
# --- Load provider and validate auth ---
async with AsyncSession(engine) as session:
provider = await _get_provider_by_token(session, token, "webhook")
provider_id = provider.id
provider_config = provider.config or {}
provider_name = provider.name
store_payloads = provider_config.get("store_payloads", True)
max_stored = min(max(int(provider_config.get("max_stored_payloads", 20)), 1), 100)
raw_body = await _read_bounded_body(request)
# Bounded read above already enforces the size cap; no need to re-check.
if not _verify_generic_webhook_auth(provider_config, request, raw_body):
raise HTTPException(status_code=403, detail="Authentication failed")
safe_headers = _filter_headers(dict(request.headers))
# Parse JSON payload from the already-bounded raw_body (request.body()
# has been consumed, so request.json() is no longer usable here).
try:
payload = json.loads(raw_body.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError("Payload must be a JSON object")
except (UnicodeDecodeError, json.JSONDecodeError, ValueError):
if store_payloads:
async with AsyncSession(get_engine()) as log_session:
await _save_webhook_log(
log_session, provider_id, request.method, safe_headers,
{}, "error", error_message="Invalid JSON", max_count=max_stored,
)
await log_session.commit()
raise HTTPException(status_code=400, detail="Invalid JSON")
# Parse via JSONPath mappings
req_headers = dict(request.headers)
event = parse_generic_webhook(payload, provider_name, provider_config, headers=req_headers)
if event is None:
if store_payloads:
async with AsyncSession(get_engine()) as log_session:
await _save_webhook_log(
log_session, provider_id, request.method, safe_headers,
payload, "unmatched", max_count=max_stored,
)
await log_session.commit()
return {"ok": True, "skipped": "parse failed"}
# Inject source IP
source_ip = request.client.host if request.client else ""
event.extra["source_ip"] = source_ip
# --- Dispatch ---
dispatched = await _dispatch_webhook_event(
engine=engine,
provider_id=provider_id,
provider_name=provider_name,
provider_config=provider_config,
event=event,
detail_keys=(
"event_type_raw", "source_ip",
),
)
# Log matched payload (separate session — dispatch already committed)
if store_payloads:
async with AsyncSession(engine) as log_session:
await _save_webhook_log(
log_session, provider_id, request.method, safe_headers,
payload, "matched" if dispatched > 0 else "unmatched",
extracted_fields=dict(event.extra),
max_count=max_stored,
)
await log_session.commit()
return {"ok": True, "dispatched": dispatched}
def _build_target_groups(
event: ServiceEvent,
link_data: list[dict[str, Any]],
provider_config: dict[str, Any],
app_tz: str = "UTC",
) -> list[tuple[Any, list[TargetConfig]]]:
"""Build TargetConfigs for dispatch, grouped by their TrackingConfig.
Targets sharing a TrackingConfig dispatch together so a single
``apply_tracking_display_filters`` pass can shape one event for the
whole group; targets with different TCs may see differently-shaped
events (e.g. one with favorites_only, one without).
"""
groups: dict[int, tuple[Any, list[TargetConfig]]] = {}
for ld in link_data:
tc = ld["tracking_config"]
if tc and not event_allowed_by_config(event, tc, app_tz):
continue
tmpl = ld["template_config"]
target_cfg = TargetConfig(
type=ld["target_type"],
config=ld["target_config"],
template_slots=ld["template_slots"],
date_format=tmpl.date_format if tmpl else "%d.%m.%Y, %H:%M UTC",
date_only_format=tmpl.date_only_format if tmpl and tmpl.date_only_format else "%d.%m.%Y",
provider_api_key=provider_config.get("api_token"),
provider_internal_url=provider_config.get("url", ""),
provider_external_url=provider_config.get("url", ""),
receivers=ld["receivers"],
)
key = id(tc) if tc is not None else 0
if key not in groups:
groups[key] = (tc, [])
groups[key][1].append(target_cfg)
return list(groups.values())
Reference in New Issue View Git Blame Copy Permalink