feat(stats): resource metrics dashboard + sites logs/stats

Background collector samples CPU/memory/network/block I/O for every instance and site on a configurable interval (default 15s, range 5-300s), persists samples to SQLite with a configurable retention window (default 2h, range 0-24h), and skips ticks gracefully when the Docker daemon is unreachable. Settings are reloadable without a restart — each tick re-reads them. New API endpoints: - GET /api/system/stats (host snapshot: info + df) - GET /api/system/stats/history - GET /api/system/stats/top?by=cpu|memory - GET /api/projects/{id}/stages/{s}/instances/{iid}/stats/history - GET /api/sites/{id}/stats[/history] - GET /api/sites/{id}/logs (SSE + JSON, reuses instance log streamer) Frontend: - ECharts added with tree-shaken imports (~180KB gzip) for future-proof time-series/gantt/graph visualizations - CollapsibleSection wraps all dashboard sections (system health, daemons, system resources, static sites, projects) with localStorage-persisted open state - SystemResourcesCard shows capacity tiles, workload utilization chart with 30m/2h/6h/24h window picker, disk breakdown with reclaimable callouts, and top 5 consumers - ContainerStats and ContainerLogs take a source discriminated union so sites reuse the same components as instances; sites detail page embeds both for Deno backend debugging - Settings › Maintenance exposes collection interval + retention - Docker-unavailable state returns 503 and renders an amber banner instead of a generic 500 Full i18n coverage (en + ru) for all new strings.
2026-04-24 15:02:43 +03:00
parent 0632f512e6
commit 05440a5f92
27 changed files with 1897 additions and 112 deletions
@@ -78,9 +78,40 @@ type Settings struct {
 	BackupEnabled       bool   `json:"backup_enabled"`
 	BackupIntervalHours  int    `json:"backup_interval_hours"`
 	BackupRetentionCount int    `json:"backup_retention_count"`
+	StatsIntervalSeconds int    `json:"stats_interval_seconds"` // 0 disables collection
+	StatsRetentionHours  int    `json:"stats_retention_hours"`  // 0 disables collection
 	UpdatedAt            string `json:"updated_at"`
 }

+// ContainerStatsSample is one persisted sample of container resource usage.
+// Cumulative counters (network, block I/O) require differencing two samples
+// to get rates; CPU is already a percent-since-previous-sample value.
+type ContainerStatsSample struct {
+	ContainerID     string `json:"container_id"`
+	OwnerType       string `json:"owner_type"` // "instance" or "site"
+	OwnerID         string `json:"owner_id"`
+	TS              int64  `json:"ts"` // Unix seconds UTC
+	CPUPercent      float64 `json:"cpu_percent"`
+	MemoryUsage     int64  `json:"memory_usage"`
+	MemoryLimit     int64  `json:"memory_limit"`
+	NetworkRxBytes  int64  `json:"network_rx_bytes"`
+	NetworkTxBytes  int64  `json:"network_tx_bytes"`
+	BlockReadBytes  int64  `json:"block_read_bytes"`
+	BlockWriteBytes int64  `json:"block_write_bytes"`
+}
+
+// SystemStatsSample is one persisted host-level snapshot that aggregates
+// workload usage across all containers plus daemon capacity + disk totals.
+type SystemStatsSample struct {
+	TS                 int64   `json:"ts"` // Unix seconds UTC
+	NCPU               int     `json:"ncpu"`
+	MemoryTotal        int64   `json:"memory_total"`
+	WorkloadCPUPercent float64 `json:"workload_cpu_percent"`
+	WorkloadMemUsage   int64   `json:"workload_mem_usage"`
+	ContainersRunning  int     `json:"containers_running"`
+	DiskTotalBytes     int64   `json:"disk_total_bytes"`
+}
+
 // Backup represents a backup metadata record.
 type Backup struct {
 	ID         string `json:"id"`
@@ -18,6 +18,7 @@ func (s *Store) GetSettings() (Settings, error) {
 		        traefik_entrypoint, traefik_cert_resolver, traefik_network, traefik_api_url,
 		        image_prune_threshold_mb,
 		        backup_enabled, backup_interval_hours, backup_retention_count,
+		        stats_interval_seconds, stats_retention_hours,
 		        updated_at
 		 FROM settings WHERE id = 1`,
 	).Scan(&st.Domain, &st.ServerIP, &st.PublicIP, &st.Network, &st.SubdomainPattern, &st.NotificationURL,
@@ -29,6 +30,7 @@ func (s *Store) GetSettings() (Settings, error) {
 		&st.TraefikEntrypoint, &st.TraefikCertResolver, &st.TraefikNetwork, &st.TraefikAPIURL,
 		&st.ImagePruneThresholdMB,
 		&backupEnabled, &st.BackupIntervalHours, &st.BackupRetentionCount,
+		&st.StatsIntervalSeconds, &st.StatsRetentionHours,
 		&st.UpdatedAt)
 	if err != nil {
 		return Settings{}, fmt.Errorf("query settings: %w", err)
@@ -65,6 +67,7 @@ func (s *Store) UpdateSettings(st Settings) error {
 			traefik_entrypoint=?, traefik_cert_resolver=?, traefik_network=?, traefik_api_url=?,
 			image_prune_threshold_mb=?,
 			backup_enabled=?, backup_interval_hours=?, backup_retention_count=?,
+			stats_interval_seconds=?, stats_retention_hours=?,
 			updated_at=?
 		 WHERE id = 1`,
 		st.Domain, st.ServerIP, st.PublicIP, st.Network, st.SubdomainPattern, st.NotificationURL,
@@ -76,6 +79,7 @@ func (s *Store) UpdateSettings(st Settings) error {
 		st.TraefikEntrypoint, st.TraefikCertResolver, st.TraefikNetwork, st.TraefikAPIURL,
 		st.ImagePruneThresholdMB,
 		backupEnabled, st.BackupIntervalHours, st.BackupRetentionCount,
+		st.StatsIntervalSeconds, st.StatsRetentionHours,
 		st.UpdatedAt,
 	)
 	if err != nil {
@@ -0,0 +1,157 @@
+package store
+
+import (
+	"fmt"
+)
+
+// InsertContainerStatsSample appends a single container sample row.
+func (s *Store) InsertContainerStatsSample(sample ContainerStatsSample) error {
+	_, err := s.db.Exec(
+		`INSERT INTO container_stats_samples (
+			container_id, owner_type, owner_id, ts,
+			cpu_percent, memory_usage, memory_limit,
+			network_rx, network_tx, block_read, block_write
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		sample.ContainerID, sample.OwnerType, sample.OwnerID, sample.TS,
+		sample.CPUPercent, sample.MemoryUsage, sample.MemoryLimit,
+		sample.NetworkRxBytes, sample.NetworkTxBytes,
+		sample.BlockReadBytes, sample.BlockWriteBytes,
+	)
+	if err != nil {
+		return fmt.Errorf("insert container stats sample: %w", err)
+	}
+	return nil
+}
+
+// InsertSystemStatsSample appends a single host-level sample row.
+func (s *Store) InsertSystemStatsSample(sample SystemStatsSample) error {
+	_, err := s.db.Exec(
+		`INSERT INTO system_stats_samples (
+			ts, ncpu, memory_total,
+			workload_cpu_percent, workload_mem_usage,
+			containers_running, disk_total_bytes
+		) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+		sample.TS, sample.NCPU, sample.MemoryTotal,
+		sample.WorkloadCPUPercent, sample.WorkloadMemUsage,
+		sample.ContainersRunning, sample.DiskTotalBytes,
+	)
+	if err != nil {
+		return fmt.Errorf("insert system stats sample: %w", err)
+	}
+	return nil
+}
+
+// ListContainerStatsSamples returns samples for the given owner since the
+// given unix timestamp (inclusive), ordered by ts ascending.
+func (s *Store) ListContainerStatsSamples(ownerType, ownerID string, sinceTS int64) ([]ContainerStatsSample, error) {
+	rows, err := s.db.Query(
+		`SELECT container_id, owner_type, owner_id, ts,
+		        cpu_percent, memory_usage, memory_limit,
+		        network_rx, network_tx, block_read, block_write
+		 FROM container_stats_samples
+		 WHERE owner_type = ? AND owner_id = ? AND ts >= ?
+		 ORDER BY ts ASC`,
+		ownerType, ownerID, sinceTS,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("list container stats samples: %w", err)
+	}
+	defer rows.Close()
+
+	var out []ContainerStatsSample
+	for rows.Next() {
+		var s ContainerStatsSample
+		if err := rows.Scan(
+			&s.ContainerID, &s.OwnerType, &s.OwnerID, &s.TS,
+			&s.CPUPercent, &s.MemoryUsage, &s.MemoryLimit,
+			&s.NetworkRxBytes, &s.NetworkTxBytes,
+			&s.BlockReadBytes, &s.BlockWriteBytes,
+		); err != nil {
+			return nil, fmt.Errorf("scan container stats sample: %w", err)
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
+
+// ListAllRecentContainerStatsSamples returns samples across every owner since
+// the given unix timestamp, ordered by ts ascending. Used by the system
+// dashboard "top containers" widget where the UI wants a mixed pool.
+func (s *Store) ListAllRecentContainerStatsSamples(sinceTS int64) ([]ContainerStatsSample, error) {
+	rows, err := s.db.Query(
+		`SELECT container_id, owner_type, owner_id, ts,
+		        cpu_percent, memory_usage, memory_limit,
+		        network_rx, network_tx, block_read, block_write
+		 FROM container_stats_samples
+		 WHERE ts >= ?
+		 ORDER BY ts ASC`,
+		sinceTS,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("list all recent container stats samples: %w", err)
+	}
+	defer rows.Close()
+
+	var out []ContainerStatsSample
+	for rows.Next() {
+		var s ContainerStatsSample
+		if err := rows.Scan(
+			&s.ContainerID, &s.OwnerType, &s.OwnerID, &s.TS,
+			&s.CPUPercent, &s.MemoryUsage, &s.MemoryLimit,
+			&s.NetworkRxBytes, &s.NetworkTxBytes,
+			&s.BlockReadBytes, &s.BlockWriteBytes,
+		); err != nil {
+			return nil, fmt.Errorf("scan container stats sample: %w", err)
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
+
+// ListSystemStatsSamples returns host samples since the given unix timestamp.
+func (s *Store) ListSystemStatsSamples(sinceTS int64) ([]SystemStatsSample, error) {
+	rows, err := s.db.Query(
+		`SELECT ts, ncpu, memory_total,
+		        workload_cpu_percent, workload_mem_usage,
+		        containers_running, disk_total_bytes
+		 FROM system_stats_samples
+		 WHERE ts >= ?
+		 ORDER BY ts ASC`,
+		sinceTS,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("list system stats samples: %w", err)
+	}
+	defer rows.Close()
+
+	var out []SystemStatsSample
+	for rows.Next() {
+		var s SystemStatsSample
+		if err := rows.Scan(
+			&s.TS, &s.NCPU, &s.MemoryTotal,
+			&s.WorkloadCPUPercent, &s.WorkloadMemUsage,
+			&s.ContainersRunning, &s.DiskTotalBytes,
+		); err != nil {
+			return nil, fmt.Errorf("scan system stats sample: %w", err)
+		}
+		out = append(out, s)
+	}
+	return out, rows.Err()
+}
+
+// PruneStatsSamplesBefore deletes all samples older than the given unix timestamp
+// from both the container and system stats tables. Returns rows deleted across
+// both tables.
+func (s *Store) PruneStatsSamplesBefore(ts int64) (int64, error) {
+	r1, err := s.db.Exec(`DELETE FROM container_stats_samples WHERE ts < ?`, ts)
+	if err != nil {
+		return 0, fmt.Errorf("prune container stats samples: %w", err)
+	}
+	r2, err := s.db.Exec(`DELETE FROM system_stats_samples WHERE ts < ?`, ts)
+	if err != nil {
+		return 0, fmt.Errorf("prune system stats samples: %w", err)
+	}
+	n1, _ := r1.RowsAffected()
+	n2, _ := r2.RowsAffected()
+	return n1 + n2, nil
+}
@@ -133,10 +133,46 @@ func (s *Store) runMigrations() error {
 		// avoid a destructive migration on SQLite.
 		`ALTER TABLE projects ADD COLUMN webhook_secret TEXT NOT NULL DEFAULT ''`,
 		`ALTER TABLE static_sites ADD COLUMN webhook_secret TEXT NOT NULL DEFAULT ''`,
+		// Resource metrics collection (2026-04-24). Interval in seconds,
+		// retention in hours. 0 in either disables collection.
+		`ALTER TABLE settings ADD COLUMN stats_interval_seconds INTEGER NOT NULL DEFAULT 15`,
+		`ALTER TABLE settings ADD COLUMN stats_retention_hours INTEGER NOT NULL DEFAULT 2`,
 	}

 	// Additive stack tables (2026-04-16). Created here rather than in the
 	// schema constant so older databases pick them up on restart.
+	statsTables := []string{
+		`CREATE TABLE IF NOT EXISTS container_stats_samples (
+			id            INTEGER PRIMARY KEY AUTOINCREMENT,
+			container_id  TEXT NOT NULL,
+			owner_type    TEXT NOT NULL,
+			owner_id      TEXT NOT NULL,
+			ts            INTEGER NOT NULL,
+			cpu_percent   REAL NOT NULL DEFAULT 0,
+			memory_usage  INTEGER NOT NULL DEFAULT 0,
+			memory_limit  INTEGER NOT NULL DEFAULT 0,
+			network_rx    INTEGER NOT NULL DEFAULT 0,
+			network_tx    INTEGER NOT NULL DEFAULT 0,
+			block_read    INTEGER NOT NULL DEFAULT 0,
+			block_write   INTEGER NOT NULL DEFAULT 0
+		)`,
+		`CREATE TABLE IF NOT EXISTS system_stats_samples (
+			id                   INTEGER PRIMARY KEY AUTOINCREMENT,
+			ts                   INTEGER NOT NULL,
+			ncpu                 INTEGER NOT NULL DEFAULT 0,
+			memory_total         INTEGER NOT NULL DEFAULT 0,
+			workload_cpu_percent REAL NOT NULL DEFAULT 0,
+			workload_mem_usage   INTEGER NOT NULL DEFAULT 0,
+			containers_running   INTEGER NOT NULL DEFAULT 0,
+			disk_total_bytes     INTEGER NOT NULL DEFAULT 0
+		)`,
+	}
+	for _, t := range statsTables {
+		if _, err := s.db.Exec(t); err != nil {
+			return fmt.Errorf("create stats table: %w", err)
+		}
+	}
+
 	stackTables := []string{
 		`CREATE TABLE IF NOT EXISTS stacks (
 			id                   TEXT PRIMARY KEY,
@@ -201,6 +237,10 @@ func (s *Store) runMigrations() error {
 		`CREATE INDEX IF NOT EXISTS idx_stack_deploys_stack_id ON stack_deploys(stack_id)`,
 		`CREATE UNIQUE INDEX IF NOT EXISTS idx_projects_webhook_secret ON projects(webhook_secret) WHERE webhook_secret != ''`,
 		`CREATE UNIQUE INDEX IF NOT EXISTS idx_static_sites_webhook_secret ON static_sites(webhook_secret) WHERE webhook_secret != ''`,
+		`CREATE INDEX IF NOT EXISTS idx_container_stats_owner_ts ON container_stats_samples(owner_type, owner_id, ts)`,
+		`CREATE INDEX IF NOT EXISTS idx_container_stats_container_ts ON container_stats_samples(container_id, ts)`,
+		`CREATE INDEX IF NOT EXISTS idx_container_stats_ts ON container_stats_samples(ts)`,
+		`CREATE INDEX IF NOT EXISTS idx_system_stats_ts ON system_stats_samples(ts)`,
 	}
 	for _, idx := range indexes {
 		if _, err := s.db.Exec(idx); err != nil {