refactor: comprehensive codebase review — security, performance, quality, UX
Security: - Fix NUT protocol command injection (validate names against safe regex) - Enable Jinja2 autoescape=True to prevent HTML injection via external data - Add WebhookProviderConfig validation model Performance: - Shared aiohttp.ClientSession singleton (replaces 40+ per-request sessions) - Fix 4 N+1 queries with batch IN loads (poller, scheduler, memory, broadcast) - asyncio.gather for Gitea commands and notification dispatcher - Add DB indexes on NotificationTrackerState.tracker_id, CommandTrackerListener - LRU cache for compiled Jinja2 templates - Daily EventLog cleanup job (90-day retention) - 30s HTTP timeout on all external calls - GROUP BY for target type counts (replaces 7 sequential queries) Code quality: - Extract get_owned_entity() helper (replaces 11 duplicate functions) - Extract slot_helpers.py (load_slots, save_slots, render_template_preview) - Extract command_utils.py (tracker lookup, last event, collection IDs) - Extract http_session.py (shared session lifecycle) - Provider connection validation dedup (3x → 1 helper) - Command dispatch tables replacing if/elif chains - Album+links fetch helper (fetch_albums_with_links) - Provider dispatch polymorphism (list_provider_collections) - Immutable _enrich_assets (no longer mutates in-place) - Fix _format_assets return type + handler unpacking Frontend: - Fix 18+ hardcoded English strings → t() with new i18n keys (en + ru) - Mobile "More" nav panel with provider filter and search - Shared Button.svelte component (4 variants, 2 sizes) - Shared ErrorBanner.svelte component (8 pages updated) - SvelteKit goto() replacing window.location.href - Dashboard grid fixed for 4 cards, paginator opacity consistency Functionality: - max_instances=1 on scheduler jobs (prevents duplicate events) - Webhook provider in watcher (prevents error spam) - Fix stale SQLModel reference in poller - Gitea get_repo() direct API call
This commit is contained in:
@@ -2,27 +2,55 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable, Coroutine
|
||||
from typing import Any
|
||||
|
||||
import aiohttp
|
||||
from sqlmodel import select
|
||||
from sqlmodel.ext.asyncio.session import AsyncSession
|
||||
|
||||
from ..database.engine import get_engine
|
||||
from ..database.models import (
|
||||
CommandConfig, CommandTracker, EventLog,
|
||||
NotificationTracker, ServiceProvider, TelegramBot,
|
||||
CommandConfig, CommandTracker, ServiceProvider, TelegramBot,
|
||||
)
|
||||
from ..services import make_gitea_provider
|
||||
from .base import ProviderCommandHandler
|
||||
from .handler import _render_cmd_template, _get_notification_trackers_for_providers
|
||||
from ..services.http_session import get_http_session
|
||||
from .base import CommandResponse, ProviderCommandHandler
|
||||
from .command_utils import get_last_event_str, get_tracked_collection_ids, get_trackers_for_provider
|
||||
from .handler import _render_cmd_template
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
_GITEA_COMMANDS = {"status", "repos", "issues", "prs", "commits"}
|
||||
|
||||
|
||||
def _get_tracked_repos(
|
||||
provider: ServiceProvider,
|
||||
trackers: list,
|
||||
) -> list[tuple[ServiceProvider, str, str]]:
|
||||
"""Get (provider, owner, repo) tuples from tracked collection_ids."""
|
||||
if not provider.config.get("api_token"):
|
||||
return []
|
||||
collection_ids = get_tracked_collection_ids(provider, trackers)
|
||||
repos: list[tuple[ServiceProvider, str, str]] = []
|
||||
for full_name in collection_ids:
|
||||
parts = full_name.split("/", 1)
|
||||
if len(parts) == 2:
|
||||
repos.append((provider, parts[0], parts[1]))
|
||||
return repos
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Command dispatch table
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TEXT_COMMANDS: dict[str, Callable[..., Coroutine[Any, Any, dict[str, Any]]]] = {}
|
||||
|
||||
|
||||
def _text_cmd(fn: Callable[..., Coroutine[Any, Any, dict[str, Any]]]) -> Callable[..., Coroutine[Any, Any, dict[str, Any]]]:
|
||||
"""Register a function in the text command dispatch table."""
|
||||
name = fn.__name__.removeprefix("_cmd_")
|
||||
_TEXT_COMMANDS[name] = fn
|
||||
return fn
|
||||
|
||||
|
||||
class GiteaCommandHandler(ProviderCommandHandler):
|
||||
"""Handles Gitea-specific bot commands."""
|
||||
|
||||
@@ -44,91 +72,35 @@ class GiteaCommandHandler(ProviderCommandHandler):
|
||||
count: int,
|
||||
locale: str,
|
||||
response_mode: str,
|
||||
providers_map: dict[int, ServiceProvider],
|
||||
provider: ServiceProvider,
|
||||
cmd_templates: dict[str, dict[str, str]],
|
||||
bot: TelegramBot,
|
||||
ctx_tuples: list[tuple[CommandTracker, CommandConfig, ServiceProvider]],
|
||||
) -> str | list[dict[str, Any]] | None:
|
||||
if cmd == "status":
|
||||
ctx = await _cmd_status(providers_map)
|
||||
return _render_cmd_template(cmd_templates, "status", locale, ctx)
|
||||
if cmd == "repos":
|
||||
ctx = await _cmd_repos(providers_map)
|
||||
return _render_cmd_template(cmd_templates, "repos", locale, ctx)
|
||||
if cmd == "issues":
|
||||
ctx = await _cmd_issues(providers_map, count)
|
||||
return _render_cmd_template(cmd_templates, "issues", locale, ctx)
|
||||
if cmd == "prs":
|
||||
ctx = await _cmd_prs(providers_map, count)
|
||||
return _render_cmd_template(cmd_templates, "prs", locale, ctx)
|
||||
if cmd == "commits":
|
||||
ctx = await _cmd_commits(providers_map, count)
|
||||
return _render_cmd_template(cmd_templates, "commits", locale, ctx)
|
||||
return None
|
||||
tracker: CommandTracker,
|
||||
config: CommandConfig,
|
||||
) -> CommandResponse | None:
|
||||
fn = _TEXT_COMMANDS.get(cmd)
|
||||
if fn is None:
|
||||
return None
|
||||
ctx = await fn(provider, count)
|
||||
return CommandResponse(text=_render_cmd_template(cmd_templates, cmd, locale, ctx))
|
||||
|
||||
|
||||
def _get_tracked_repos(
|
||||
providers_map: dict[int, ServiceProvider],
|
||||
trackers: list[NotificationTracker],
|
||||
) -> list[tuple[ServiceProvider, str, str]]:
|
||||
"""Get (provider, owner, repo) tuples from tracked collection_ids."""
|
||||
repos: list[tuple[ServiceProvider, str, str]] = []
|
||||
for tracker in trackers:
|
||||
provider = providers_map.get(tracker.provider_id)
|
||||
if not provider or provider.type != "gitea":
|
||||
continue
|
||||
if not provider.config.get("api_token"):
|
||||
continue
|
||||
for full_name in (tracker.collection_ids or []):
|
||||
parts = full_name.split("/", 1)
|
||||
if len(parts) == 2:
|
||||
repos.append((provider, parts[0], parts[1]))
|
||||
# Also check filters.collections
|
||||
for tracker in trackers:
|
||||
provider = providers_map.get(tracker.provider_id)
|
||||
if not provider or provider.type != "gitea":
|
||||
continue
|
||||
if not provider.config.get("api_token"):
|
||||
continue
|
||||
for full_name in (tracker.filters or {}).get("collections", []):
|
||||
parts = full_name.split("/", 1)
|
||||
if len(parts) == 2:
|
||||
entry = (provider, parts[0], parts[1])
|
||||
if entry not in repos:
|
||||
repos.append(entry)
|
||||
return repos[:20] # Cap to prevent API hammering
|
||||
@_text_cmd
|
||||
async def _cmd_status(provider: ServiceProvider, count: int) -> dict[str, Any]:
|
||||
trackers = await get_trackers_for_provider(provider.id)
|
||||
tracked_repos = _get_tracked_repos(provider, trackers)
|
||||
|
||||
|
||||
async def _cmd_status(providers_map: dict[int, ServiceProvider]) -> dict[str, Any]:
|
||||
provider_ids = set(providers_map.keys())
|
||||
trackers = await _get_notification_trackers_for_providers(provider_ids)
|
||||
tracked_repos = _get_tracked_repos(providers_map, trackers)
|
||||
|
||||
# Get server version from first Gitea provider with token
|
||||
# Get server version
|
||||
server_version = "unknown"
|
||||
async with aiohttp.ClientSession() as http:
|
||||
for provider in providers_map.values():
|
||||
if provider.type == "gitea" and provider.config.get("api_token"):
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
version = await gitea.client.get_server_version()
|
||||
if version:
|
||||
server_version = version
|
||||
break
|
||||
if provider.config.get("api_token"):
|
||||
http = await get_http_session()
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
version = await gitea.client.get_server_version()
|
||||
if version:
|
||||
server_version = version
|
||||
|
||||
# Last event
|
||||
engine = get_engine()
|
||||
async with AsyncSession(engine) as session:
|
||||
tracker_ids = [t.id for t in trackers]
|
||||
if tracker_ids:
|
||||
result = await session.exec(
|
||||
select(EventLog)
|
||||
.where(EventLog.tracker_id.in_(tracker_ids))
|
||||
.order_by(EventLog.created_at.desc()).limit(1)
|
||||
)
|
||||
last_event = result.first()
|
||||
else:
|
||||
last_event = None
|
||||
last_str = last_event.created_at.strftime("%Y-%m-%d %H:%M") if last_event else "-"
|
||||
tracker_ids = [t.id for t in trackers]
|
||||
last_str = await get_last_event_str(tracker_ids)
|
||||
|
||||
return {
|
||||
"repos_count": len(tracked_repos),
|
||||
@@ -137,116 +109,139 @@ async def _cmd_status(providers_map: dict[int, ServiceProvider]) -> dict[str, An
|
||||
}
|
||||
|
||||
|
||||
async def _cmd_repos(providers_map: dict[int, ServiceProvider]) -> dict[str, Any]:
|
||||
provider_ids = set(providers_map.keys())
|
||||
trackers = await _get_notification_trackers_for_providers(provider_ids)
|
||||
tracked_repos = _get_tracked_repos(providers_map, trackers)
|
||||
@_text_cmd
|
||||
async def _cmd_repos(provider: ServiceProvider, count: int) -> dict[str, Any]:
|
||||
trackers = await get_trackers_for_provider(provider.id)
|
||||
tracked_repos = _get_tracked_repos(provider, trackers)
|
||||
|
||||
repos_data: list[dict[str, Any]] = []
|
||||
async with aiohttp.ClientSession() as http:
|
||||
for provider, owner, repo in tracked_repos:
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
try:
|
||||
all_repos = await gitea.client.get_repos(limit=50)
|
||||
for r in all_repos:
|
||||
if r.get("full_name") == f"{owner}/{repo}":
|
||||
repos_data.append({
|
||||
"full_name": r.get("full_name", ""),
|
||||
"description": r.get("description", ""),
|
||||
"stars": r.get("stars_count", 0),
|
||||
"url": r.get("html_url", ""),
|
||||
})
|
||||
break
|
||||
else:
|
||||
repos_data.append({
|
||||
"full_name": f"{owner}/{repo}",
|
||||
"description": "",
|
||||
"stars": 0,
|
||||
"url": "",
|
||||
})
|
||||
except Exception:
|
||||
repos_data.append({
|
||||
"full_name": f"{owner}/{repo}",
|
||||
"description": "?",
|
||||
"stars": 0,
|
||||
"url": "",
|
||||
})
|
||||
http = await get_http_session()
|
||||
|
||||
async def _fetch_repo(prov: ServiceProvider, owner: str, repo: str) -> dict[str, Any]:
|
||||
gitea = make_gitea_provider(http, prov)
|
||||
# Use direct get_repo endpoint instead of listing all repos
|
||||
r = await gitea.client.get_repo(owner, repo)
|
||||
if r:
|
||||
return {
|
||||
"full_name": r.get("full_name", ""),
|
||||
"description": r.get("description", ""),
|
||||
"stars": r.get("stars_count", 0),
|
||||
"url": r.get("html_url", ""),
|
||||
}
|
||||
return {
|
||||
"full_name": f"{owner}/{repo}",
|
||||
"description": "",
|
||||
"stars": 0,
|
||||
"url": "",
|
||||
}
|
||||
|
||||
tasks = [_fetch_repo(prov, owner, repo) for prov, owner, repo in tracked_repos]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for (prov, owner, repo), result in zip(tracked_repos, results):
|
||||
if isinstance(result, Exception):
|
||||
_LOGGER.warning("Failed to fetch repo %s/%s: %s", owner, repo, result)
|
||||
repos_data.append({
|
||||
"full_name": f"{owner}/{repo}",
|
||||
"description": "?",
|
||||
"stars": 0,
|
||||
"url": "",
|
||||
})
|
||||
else:
|
||||
repos_data.append(result)
|
||||
|
||||
return {"repos": repos_data}
|
||||
|
||||
|
||||
async def _cmd_issues(
|
||||
providers_map: dict[int, ServiceProvider], count: int,
|
||||
) -> dict[str, Any]:
|
||||
provider_ids = set(providers_map.keys())
|
||||
trackers = await _get_notification_trackers_for_providers(provider_ids)
|
||||
tracked_repos = _get_tracked_repos(providers_map, trackers)
|
||||
@_text_cmd
|
||||
async def _cmd_issues(provider: ServiceProvider, count: int) -> dict[str, Any]:
|
||||
trackers = await get_trackers_for_provider(provider.id)
|
||||
tracked_repos = _get_tracked_repos(provider, trackers)
|
||||
|
||||
all_issues: list[dict[str, Any]] = []
|
||||
async with aiohttp.ClientSession() as http:
|
||||
for provider, owner, repo in tracked_repos:
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
issues = await gitea.client.get_repo_issues(owner, repo, limit=count)
|
||||
for issue in issues:
|
||||
all_issues.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"number": issue.get("number", 0),
|
||||
"title": issue.get("title", ""),
|
||||
"url": issue.get("html_url", ""),
|
||||
"user": issue.get("user", {}).get("login", ""),
|
||||
"state": issue.get("state", ""),
|
||||
})
|
||||
http = await get_http_session()
|
||||
|
||||
async def _fetch_issues(prov: ServiceProvider, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
gitea = make_gitea_provider(http, prov)
|
||||
return await gitea.client.get_repo_issues(owner, repo, limit=count)
|
||||
|
||||
tasks = [_fetch_issues(prov, owner, repo) for prov, owner, repo in tracked_repos]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for (prov, owner, repo), result in zip(tracked_repos, results):
|
||||
if isinstance(result, Exception):
|
||||
_LOGGER.warning("Failed to fetch issues for %s/%s: %s", owner, repo, result)
|
||||
continue
|
||||
for issue in result:
|
||||
all_issues.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"number": issue.get("number", 0),
|
||||
"title": issue.get("title", ""),
|
||||
"url": issue.get("html_url", ""),
|
||||
"user": issue.get("user", {}).get("login", ""),
|
||||
"state": issue.get("state", ""),
|
||||
})
|
||||
|
||||
all_issues.sort(key=lambda i: i.get("number", 0), reverse=True)
|
||||
return {"issues": all_issues[:count]}
|
||||
|
||||
|
||||
async def _cmd_prs(
|
||||
providers_map: dict[int, ServiceProvider], count: int,
|
||||
) -> dict[str, Any]:
|
||||
provider_ids = set(providers_map.keys())
|
||||
trackers = await _get_notification_trackers_for_providers(provider_ids)
|
||||
tracked_repos = _get_tracked_repos(providers_map, trackers)
|
||||
@_text_cmd
|
||||
async def _cmd_prs(provider: ServiceProvider, count: int) -> dict[str, Any]:
|
||||
trackers = await get_trackers_for_provider(provider.id)
|
||||
tracked_repos = _get_tracked_repos(provider, trackers)
|
||||
|
||||
all_prs: list[dict[str, Any]] = []
|
||||
async with aiohttp.ClientSession() as http:
|
||||
for provider, owner, repo in tracked_repos:
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
prs = await gitea.client.get_repo_pulls(owner, repo, limit=count)
|
||||
for pr in prs:
|
||||
all_prs.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"number": pr.get("number", 0),
|
||||
"title": pr.get("title", ""),
|
||||
"url": pr.get("html_url", ""),
|
||||
"user": pr.get("user", {}).get("login", ""),
|
||||
"state": pr.get("state", ""),
|
||||
})
|
||||
http = await get_http_session()
|
||||
|
||||
async def _fetch_prs(prov: ServiceProvider, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
gitea = make_gitea_provider(http, prov)
|
||||
return await gitea.client.get_repo_pulls(owner, repo, limit=count)
|
||||
|
||||
tasks = [_fetch_prs(prov, owner, repo) for prov, owner, repo in tracked_repos]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for (prov, owner, repo), result in zip(tracked_repos, results):
|
||||
if isinstance(result, Exception):
|
||||
_LOGGER.warning("Failed to fetch PRs for %s/%s: %s", owner, repo, result)
|
||||
continue
|
||||
for pr in result:
|
||||
all_prs.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"number": pr.get("number", 0),
|
||||
"title": pr.get("title", ""),
|
||||
"url": pr.get("html_url", ""),
|
||||
"user": pr.get("user", {}).get("login", ""),
|
||||
"state": pr.get("state", ""),
|
||||
})
|
||||
|
||||
all_prs.sort(key=lambda p: p.get("number", 0), reverse=True)
|
||||
return {"prs": all_prs[:count]}
|
||||
|
||||
|
||||
async def _cmd_commits(
|
||||
providers_map: dict[int, ServiceProvider], count: int,
|
||||
) -> dict[str, Any]:
|
||||
provider_ids = set(providers_map.keys())
|
||||
trackers = await _get_notification_trackers_for_providers(provider_ids)
|
||||
tracked_repos = _get_tracked_repos(providers_map, trackers)
|
||||
@_text_cmd
|
||||
async def _cmd_commits(provider: ServiceProvider, count: int) -> dict[str, Any]:
|
||||
trackers = await get_trackers_for_provider(provider.id)
|
||||
tracked_repos = _get_tracked_repos(provider, trackers)
|
||||
|
||||
all_commits: list[dict[str, Any]] = []
|
||||
async with aiohttp.ClientSession() as http:
|
||||
for provider, owner, repo in tracked_repos:
|
||||
gitea = make_gitea_provider(http, provider)
|
||||
commits = await gitea.client.get_repo_commits(owner, repo, limit=count)
|
||||
for c in commits:
|
||||
commit_data = c.get("commit", {})
|
||||
all_commits.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"short_id": c.get("sha", "")[:7],
|
||||
"message": commit_data.get("message", "").split("\n")[0][:80],
|
||||
"author": commit_data.get("author", {}).get("name", ""),
|
||||
"url": c.get("html_url", ""),
|
||||
})
|
||||
http = await get_http_session()
|
||||
|
||||
async def _fetch_commits(prov: ServiceProvider, owner: str, repo: str) -> list[dict[str, Any]]:
|
||||
gitea = make_gitea_provider(http, prov)
|
||||
return await gitea.client.get_repo_commits(owner, repo, limit=count)
|
||||
|
||||
tasks = [_fetch_commits(prov, owner, repo) for prov, owner, repo in tracked_repos]
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
for (prov, owner, repo), result in zip(tracked_repos, results):
|
||||
if isinstance(result, Exception):
|
||||
_LOGGER.warning("Failed to fetch commits for %s/%s: %s", owner, repo, result)
|
||||
continue
|
||||
for c in result:
|
||||
commit_data = c.get("commit", {})
|
||||
all_commits.append({
|
||||
"repo": f"{owner}/{repo}",
|
||||
"short_id": c.get("sha", "")[:7],
|
||||
"message": commit_data.get("message", "").split("\n")[0][:80],
|
||||
"author": commit_data.get("author", {}).get("name", ""),
|
||||
"url": c.get("html_url", ""),
|
||||
})
|
||||
|
||||
return {"commits": all_commits[:count]}
|
||||
|
||||
Reference in New Issue
Block a user