feat(observability): phase 8 - container stats, notifications & dashboard
Add container monitoring and notification system: - Docker Stats API: real-time CPU/memory for running containers - Webhook notifications for errors (deploy failures, stale, proxy unhealthy) - Event log auto-pruning (daily, 30-day retention) - ContainerStats component with auto-polling progress bars - SystemHealthCard dashboard widget with running/proxy/error counts - Full EN/RU i18n for stats and system health
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import type {
|
||||
ApiEnvelope,
|
||||
ContainerStats,
|
||||
Deploy,
|
||||
DeployLog,
|
||||
EventLogEntry,
|
||||
@@ -420,4 +421,16 @@ export function bulkCleanupStaleContainers(): Promise<{ deleted: number }> {
|
||||
return post<{ deleted: number }>('/api/containers/stale/cleanup');
|
||||
}
|
||||
|
||||
// ── Container Stats ────────────────────────────────────────────────
|
||||
|
||||
export function fetchContainerStats(
|
||||
projectId: string,
|
||||
stageId: string,
|
||||
instanceId: string
|
||||
): Promise<ContainerStats> {
|
||||
return get<ContainerStats>(
|
||||
`/api/projects/${projectId}/stages/${stageId}/instances/${instanceId}/stats`
|
||||
);
|
||||
}
|
||||
|
||||
export { ApiError };
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
<!--
|
||||
Compact CPU/memory stats bars for embedding in instance cards.
|
||||
-->
|
||||
<script lang="ts">
|
||||
import type { ContainerStats } from '$lib/types';
|
||||
import * as api from '$lib/api';
|
||||
import { t } from '$lib/i18n';
|
||||
|
||||
interface Props {
|
||||
projectId: string;
|
||||
stageId: string;
|
||||
instanceId: string;
|
||||
}
|
||||
|
||||
const { projectId, stageId, instanceId }: Props = $props();
|
||||
|
||||
let stats = $state<ContainerStats | null>(null);
|
||||
let error = $state(false);
|
||||
|
||||
$effect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const result = await api.fetchContainerStats(projectId, stageId, instanceId);
|
||||
if (!cancelled) {
|
||||
stats = result;
|
||||
error = false;
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) {
|
||||
error = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
load();
|
||||
|
||||
// Poll every 10 seconds.
|
||||
const interval = setInterval(load, 10_000);
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(interval);
|
||||
};
|
||||
});
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (bytes < 1024) return `${bytes} B`;
|
||||
const kb = bytes / 1024;
|
||||
if (kb < 1024) return `${kb.toFixed(0)} KB`;
|
||||
const mb = kb / 1024;
|
||||
if (mb < 1024) return `${mb.toFixed(1)} MB`;
|
||||
const gb = mb / 1024;
|
||||
return `${gb.toFixed(2)} GB`;
|
||||
}
|
||||
|
||||
const cpuColor = $derived(() => {
|
||||
if (!stats) return 'bg-gray-300';
|
||||
if (stats.cpu_percent > 80) return 'bg-red-500';
|
||||
if (stats.cpu_percent > 50) return 'bg-amber-500';
|
||||
return 'bg-emerald-500';
|
||||
});
|
||||
|
||||
const memColor = $derived(() => {
|
||||
if (!stats) return 'bg-gray-300';
|
||||
if (stats.memory_percent > 80) return 'bg-red-500';
|
||||
if (stats.memory_percent > 50) return 'bg-amber-500';
|
||||
return 'bg-blue-500';
|
||||
});
|
||||
</script>
|
||||
|
||||
{#if stats}
|
||||
<div class="mt-2 space-y-1">
|
||||
<!-- CPU bar -->
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="w-8 text-[10px] font-medium text-[var(--text-tertiary)]">{$t('stats.cpu')}</span>
|
||||
<div class="relative h-1.5 flex-1 overflow-hidden rounded-full bg-[var(--surface-card-hover)]">
|
||||
<div
|
||||
class="absolute inset-y-0 left-0 rounded-full transition-all duration-500 {cpuColor()}"
|
||||
style="width: {Math.min(stats.cpu_percent, 100)}%"
|
||||
></div>
|
||||
</div>
|
||||
<span class="w-10 text-right text-[10px] tabular-nums text-[var(--text-tertiary)]">
|
||||
{stats.cpu_percent.toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<!-- Memory bar -->
|
||||
<div class="flex items-center gap-2">
|
||||
<span class="w-8 text-[10px] font-medium text-[var(--text-tertiary)]">{$t('stats.mem')}</span>
|
||||
<div class="relative h-1.5 flex-1 overflow-hidden rounded-full bg-[var(--surface-card-hover)]">
|
||||
<div
|
||||
class="absolute inset-y-0 left-0 rounded-full transition-all duration-500 {memColor()}"
|
||||
style="width: {Math.min(stats.memory_percent, 100)}%"
|
||||
></div>
|
||||
</div>
|
||||
<span class="w-24 text-right text-[10px] tabular-nums text-[var(--text-tertiary)]">
|
||||
{formatBytes(stats.memory_usage)} / {formatBytes(stats.memory_limit)}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
{:else if error}
|
||||
<p class="mt-2 text-[10px] text-[var(--text-tertiary)]">{$t('stats.unavailable')}</p>
|
||||
{/if}
|
||||
@@ -4,6 +4,7 @@
|
||||
<script lang="ts">
|
||||
import type { Instance } from '$lib/types';
|
||||
import StatusBadge from './StatusBadge.svelte';
|
||||
import ContainerStats from './ContainerStats.svelte';
|
||||
import ConfirmDialog from './ConfirmDialog.svelte';
|
||||
import { IconPlay, IconStop, IconRestart, IconTrash, IconExternalLink } from '$lib/components/icons';
|
||||
import { t } from '$lib/i18n';
|
||||
@@ -141,6 +142,10 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{#if instance.status === 'running'}
|
||||
<ContainerStats projectId={projectId} stageId={instance.stage_id} instanceId={instance.id} />
|
||||
{/if}
|
||||
|
||||
{#if error}
|
||||
<p class="mt-2 text-xs text-[var(--color-danger)]">{error}</p>
|
||||
{/if}
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
<!--
|
||||
Dashboard summary card: container counts, proxy health, recent errors.
|
||||
-->
|
||||
<script lang="ts">
|
||||
import type { Instance, ProxyView, EventLogStats } from '$lib/types';
|
||||
import * as api from '$lib/api';
|
||||
import { IconServer, IconProxies, IconAlert } from '$lib/components/icons';
|
||||
import { t } from '$lib/i18n';
|
||||
|
||||
let runningCount = $state(0);
|
||||
let stoppedCount = $state(0);
|
||||
let healthyProxies = $state(0);
|
||||
let unhealthyProxies = $state(0);
|
||||
let recentErrors = $state(0);
|
||||
let loading = $state(true);
|
||||
|
||||
$effect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
async function load() {
|
||||
try {
|
||||
const [projects, proxies, eventStats] = await Promise.all([
|
||||
api.listProjects(),
|
||||
api.listAllProxies().catch(() => [] as ProxyView[]),
|
||||
api.fetchEventLogStats().catch(() => ({ info: 0, warn: 0, error: 0, total: 0 }) as EventLogStats)
|
||||
]);
|
||||
|
||||
// Gather all instances across projects/stages.
|
||||
const allInstances: Instance[] = [];
|
||||
for (const project of projects) {
|
||||
try {
|
||||
const detail = await api.getProject(project.id);
|
||||
for (const stage of detail.stages ?? []) {
|
||||
const instances = await api.listInstances(project.id, stage.id);
|
||||
allInstances.push(...instances);
|
||||
}
|
||||
} catch {
|
||||
// Skip projects that fail to load.
|
||||
}
|
||||
}
|
||||
|
||||
if (!cancelled) {
|
||||
runningCount = allInstances.filter((i) => i.status === 'running').length;
|
||||
stoppedCount = allInstances.filter((i) => i.status !== 'running').length;
|
||||
healthyProxies = proxies.filter((p) => p.health_status === 'healthy').length;
|
||||
unhealthyProxies = proxies.filter((p) => p.health_status === 'unhealthy').length;
|
||||
recentErrors = eventStats.error;
|
||||
loading = false;
|
||||
}
|
||||
} catch {
|
||||
if (!cancelled) {
|
||||
loading = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
load();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
});
|
||||
</script>
|
||||
|
||||
{#if !loading}
|
||||
<div class="rounded-xl border border-[var(--border-primary)] bg-[var(--surface-card)] p-5 shadow-[var(--shadow-sm)]">
|
||||
<h3 class="mb-4 text-sm font-semibold text-[var(--text-primary)]">{$t('systemHealth.title')}</h3>
|
||||
<div class="grid grid-cols-1 gap-3 sm:grid-cols-3">
|
||||
<!-- Containers -->
|
||||
<a href="/projects" class="flex items-center gap-3 rounded-lg p-3 transition-colors hover:bg-[var(--surface-card-hover)]">
|
||||
<div class="flex h-9 w-9 items-center justify-center rounded-lg bg-emerald-50 text-emerald-600">
|
||||
<IconServer size={18} />
|
||||
</div>
|
||||
<div>
|
||||
<p class="text-xs text-[var(--text-secondary)]">{$t('systemHealth.containers')}</p>
|
||||
<p class="text-sm font-semibold text-[var(--text-primary)]">
|
||||
<span class="text-emerald-600">{runningCount}</span>
|
||||
<span class="text-[var(--text-tertiary)]"> / </span>
|
||||
<span class="text-[var(--text-tertiary)]">{stoppedCount}</span>
|
||||
</p>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<!-- Proxies -->
|
||||
<a href="/proxies" class="flex items-center gap-3 rounded-lg p-3 transition-colors hover:bg-[var(--surface-card-hover)]">
|
||||
<div class="flex h-9 w-9 items-center justify-center rounded-lg {unhealthyProxies > 0 ? 'bg-red-50 text-red-600' : 'bg-blue-50 text-blue-600'}">
|
||||
<IconProxies size={18} />
|
||||
</div>
|
||||
<div>
|
||||
<p class="text-xs text-[var(--text-secondary)]">{$t('systemHealth.proxies')}</p>
|
||||
<p class="text-sm font-semibold text-[var(--text-primary)]">
|
||||
<span class="text-emerald-600">{healthyProxies}</span>
|
||||
{#if unhealthyProxies > 0}
|
||||
<span class="text-[var(--text-tertiary)]"> / </span>
|
||||
<span class="text-red-600">{unhealthyProxies}</span>
|
||||
{/if}
|
||||
</p>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
<!-- Recent errors -->
|
||||
<a href="/events" class="flex items-center gap-3 rounded-lg p-3 transition-colors hover:bg-[var(--surface-card-hover)]">
|
||||
<div class="flex h-9 w-9 items-center justify-center rounded-lg {recentErrors > 0 ? 'bg-red-50 text-red-600' : 'bg-gray-50 text-gray-400'}">
|
||||
<IconAlert size={18} />
|
||||
</div>
|
||||
<div>
|
||||
<p class="text-xs text-[var(--text-secondary)]">{$t('systemHealth.recentErrors')}</p>
|
||||
<p class="text-sm font-semibold {recentErrors > 0 ? 'text-red-600' : 'text-[var(--text-primary)]'}">{recentErrors}</p>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
@@ -504,6 +504,17 @@
|
||||
},
|
||||
"metadata": "Details"
|
||||
},
|
||||
"stats": {
|
||||
"cpu": "CPU",
|
||||
"mem": "MEM",
|
||||
"unavailable": "Stats unavailable"
|
||||
},
|
||||
"systemHealth": {
|
||||
"title": "System Health",
|
||||
"containers": "Containers",
|
||||
"proxies": "Proxies",
|
||||
"recentErrors": "Recent Errors"
|
||||
},
|
||||
"language": {
|
||||
"en": "English",
|
||||
"ru": "Russian"
|
||||
|
||||
@@ -504,6 +504,17 @@
|
||||
},
|
||||
"metadata": "Подробности"
|
||||
},
|
||||
"stats": {
|
||||
"cpu": "ЦП",
|
||||
"mem": "ОЗУ",
|
||||
"unavailable": "Статистика недоступна"
|
||||
},
|
||||
"systemHealth": {
|
||||
"title": "Состояние системы",
|
||||
"containers": "Контейнеры",
|
||||
"proxies": "Прокси",
|
||||
"recentErrors": "Недавние ошибки"
|
||||
},
|
||||
"language": {
|
||||
"en": "Английский",
|
||||
"ru": "Русский"
|
||||
|
||||
@@ -234,6 +234,14 @@ export interface ValidationResult {
|
||||
steps: ValidationStep[];
|
||||
}
|
||||
|
||||
/** Container CPU and memory stats from the Docker stats API. */
|
||||
export interface ContainerStats {
|
||||
cpu_percent: number;
|
||||
memory_usage: number;
|
||||
memory_limit: number;
|
||||
memory_percent: number;
|
||||
}
|
||||
|
||||
/** Unified view of standalone + deploy-managed proxies (from /api/proxies/all). */
|
||||
export interface ProxyView {
|
||||
id: string;
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import ProjectCard from '$lib/components/ProjectCard.svelte';
|
||||
import SkeletonCard from '$lib/components/SkeletonCard.svelte';
|
||||
import EmptyState from '$lib/components/EmptyState.svelte';
|
||||
import SystemHealthCard from '$lib/components/SystemHealthCard.svelte';
|
||||
import { IconDeploy, IconBox, IconServer, IconAlert, IconClock } from '$lib/components/icons';
|
||||
import { t } from '$lib/i18n';
|
||||
|
||||
@@ -124,6 +125,9 @@
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<!-- System health summary -->
|
||||
<SystemHealthCard />
|
||||
|
||||
<!-- Project cards -->
|
||||
<div>
|
||||
<h2 class="text-lg font-semibold text-[var(--text-primary)]">{$t('dashboard.projects')}</h2>
|
||||
|
||||
Reference in New Issue
Block a user