fix: resolve ERR_INSUFFICIENT_RESOURCES connection exhaustion

- Add concurrency limiter (max 4 GET requests) to API layer, leaving
  slots for SSE and health checks. Write ops bypass the limiter.
- Add AbortController to ContainerStats, project detail page, and
  dashboard to cancel in-flight requests on navigation/unmount.
- Move global SSE connection from layout to events page (only consumer).
- Add 30s heartbeat to SSE endpoint to detect zombie connections.
- Serialize dashboard project fetches to avoid parallel burst.
- Rebuild frontend in dev-server.sh so go:embed stays in sync.
This commit is contained in:
2026-04-13 00:12:14 +03:00
parent 791cd4d6af
commit 96fd910603
7 changed files with 233 additions and 87 deletions
+9
View File
@@ -7,6 +7,7 @@ import (
"log/slog"
"net/http"
"strings"
"time"
"github.com/go-chi/chi/v5"
@@ -158,6 +159,10 @@ func (s *Server) streamEvents(w http.ResponseWriter, r *http.Request) {
})
defer s.eventBus.Unsubscribe(sub)
// Periodic heartbeat so the browser detects dead connections.
heartbeat := time.NewTicker(30 * time.Second)
defer heartbeat.Stop()
ctx := r.Context()
for {
select {
@@ -168,6 +173,10 @@ func (s *Server) streamEvents(w http.ResponseWriter, r *http.Request) {
return
}
writeSSE(w, flusher, evt)
case <-heartbeat.C:
// SSE comment line — keeps the connection alive without triggering onmessage.
fmt.Fprintf(w, ": heartbeat\n\n")
flusher.Flush()
}
}
}
+4
View File
@@ -32,6 +32,10 @@ export ENCRYPTION_KEY
export ADMIN_PASSWORD="${ADMIN_PASSWORD:-admin123}"
export LISTEN_ADDR="${PORT}"
# Rebuild frontend so go:embed picks up changes.
echo "Building frontend..."
(cd web && npm run build --silent)
echo "Starting Tinyforge on http://localhost:${PORT_NUM}"
echo "Login: admin / ${ADMIN_PASSWORD}"
exec go run ./cmd/server
+84 -23
View File
@@ -43,7 +43,58 @@ class ApiError extends Error {
import { getAuthToken, clearAuth } from './auth';
// ── Concurrency limiter ────────────────────────────────────────────
// Chrome allows only 6 concurrent HTTP/1.1 connections per host.
// Reserve slots for the persistent SSE stream and health checks by
// capping regular API requests to 4 concurrent.
const MAX_CONCURRENT = 4;
let inflight = 0;
const queue: Array<() => void> = [];
function acquireSlot(signal?: AbortSignal | null): Promise<void> {
if (inflight < MAX_CONCURRENT) {
inflight++;
return Promise.resolve();
}
return new Promise<void>((resolve, reject) => {
const entry = () => { inflight++; resolve(); };
queue.push(entry);
signal?.addEventListener('abort', () => {
const idx = queue.indexOf(entry);
if (idx !== -1) {
queue.splice(idx, 1);
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
}
}, { once: true });
});
}
function releaseSlot(): void {
const next = queue.shift();
if (next) {
next();
} else {
inflight--;
}
}
async function request<T>(path: string, init?: RequestInit): Promise<T> {
// Write operations (user-initiated) bypass the concurrency limiter
// so they are never blocked behind background polling.
const method = init?.method?.toUpperCase() ?? 'GET';
if (method !== 'GET') {
return requestInner<T>(path, init);
}
await acquireSlot(init?.signal);
try {
return await requestInner<T>(path, init);
} finally {
releaseSlot();
}
}
async function requestInner<T>(path: string, init?: RequestInit): Promise<T> {
const token = getAuthToken();
const headers: Record<string, string> = {
'Content-Type': 'application/json',
@@ -82,8 +133,8 @@ async function request<T>(path: string, init?: RequestInit): Promise<T> {
return envelope.data as T;
}
function get<T>(path: string): Promise<T> {
return request<T>(path);
function get<T>(path: string, signal?: AbortSignal): Promise<T> {
return request<T>(path, signal ? { signal } : undefined);
}
function post<T>(path: string, body?: unknown): Promise<T> {
@@ -106,12 +157,12 @@ function del<T>(path: string): Promise<T> {
// ── Projects ────────────────────────────────────────────────────────
export function listProjects(): Promise<Project[]> {
return get<Project[]>('/api/projects');
export function listProjects(signal?: AbortSignal): Promise<Project[]> {
return get<Project[]>('/api/projects', signal);
}
export function getProject(id: string): Promise<ProjectDetail> {
return get<ProjectDetail>(`/api/projects/${id}`);
export function getProject(id: string, signal?: AbortSignal): Promise<ProjectDetail> {
return get<ProjectDetail>(`/api/projects/${id}`, signal);
}
export function createProject(data: Partial<Project>): Promise<Project> {
@@ -142,8 +193,8 @@ export function deleteStage(projectId: string, stageId: string): Promise<void> {
// ── Instances ───────────────────────────────────────────────────────
export function listInstances(projectId: string, stageId: string): Promise<Instance[]> {
return get<Instance[]>(`/api/projects/${projectId}/stages/${stageId}/instances`);
export function listInstances(projectId: string, stageId: string, signal?: AbortSignal): Promise<Instance[]> {
return get<Instance[]>(`/api/projects/${projectId}/stages/${stageId}/instances`, signal);
}
export function deployInstance(
@@ -198,8 +249,8 @@ export function restartInstance(
// ── Deploys ─────────────────────────────────────────────────────────
export function listDeploys(limit = 50): Promise<Deploy[]> {
return get<Deploy[]>(`/api/deploys?limit=${limit}`);
export function listDeploys(limit = 50, signal?: AbortSignal): Promise<Deploy[]> {
return get<Deploy[]>(`/api/deploys?limit=${limit}`, signal);
}
export function getDeployLogs(deployId: string): Promise<DeployLog[]> {
@@ -246,7 +297,9 @@ export function testRegistry(id: string): Promise<{ status: string }> {
}
export function listRegistryTags(registryId: string, image: string): Promise<string[]> {
return get<string[]>(`/api/registries/${registryId}/tags/${encodeURIComponent(image)}`);
// Image contains slashes (e.g. "owner/name") — don't encode them;
// the backend route uses a wildcard (/tags/*) that expects path segments.
return get<string[]>(`/api/registries/${registryId}/tags/${image}`);
}
export function listRegistryImages(registryId: string): Promise<RegistryImage[]> {
@@ -255,8 +308,8 @@ export function listRegistryImages(registryId: string): Promise<RegistryImage[]>
// ── Settings ────────────────────────────────────────────────────────
export function getSettings(): Promise<Settings> {
return get<Settings>('/api/settings');
export function getSettings(signal?: AbortSignal): Promise<Settings> {
return get<Settings>('/api/settings', signal);
}
export function updateSettings(data: Partial<Settings>): Promise<Settings> {
@@ -285,14 +338,14 @@ export function fetchContainerLogs(
return get<string[]>(`/api/projects/${projectId}/stages/${stageId}/instances/${instanceId}/logs?tail=${tail}`);
}
export function listProjectImages(projectId: string): Promise<LocalImage[]> {
return get<LocalImage[]>(`/api/projects/${projectId}/images`);
export function listProjectImages(projectId: string, signal?: AbortSignal): Promise<LocalImage[]> {
return get<LocalImage[]>(`/api/projects/${projectId}/images`, signal);
}
export function getUnusedImageStats(): Promise<{
export function getUnusedImageStats(signal?: AbortSignal): Promise<{
total_size_mb: number; count: number; threshold_mb: number; exceeded: boolean;
}> {
return get<{ total_size_mb: number; count: number; threshold_mb: number; exceeded: boolean }>('/api/docker/unused-images');
return get<{ total_size_mb: number; count: number; threshold_mb: number; exceeded: boolean }>('/api/docker/unused-images', signal);
}
export function pruneImages(): Promise<{ images_removed: number; space_reclaimed_mb: number }> {
@@ -580,8 +633,8 @@ export function clearAllEvents(): Promise<{ status: string; count: number }> {
// ── Stale Containers ────────────────────────────────────────────────
export function fetchStaleContainers(): Promise<StaleContainer[]> {
return get<StaleContainer[]>('/api/containers/stale');
export function fetchStaleContainers(signal?: AbortSignal): Promise<StaleContainer[]> {
return get<StaleContainer[]>('/api/containers/stale', signal);
}
export function cleanupStaleContainer(id: string): Promise<{ deleted: string }> {
@@ -597,10 +650,12 @@ export function bulkCleanupStaleContainers(): Promise<{ deleted: number }> {
export function fetchContainerStats(
projectId: string,
stageId: string,
instanceId: string
instanceId: string,
signal?: AbortSignal
): Promise<ContainerStats> {
return get<ContainerStats>(
`/api/projects/${projectId}/stages/${stageId}/instances/${instanceId}/stats`
`/api/projects/${projectId}/stages/${stageId}/instances/${instanceId}/stats`,
signal
);
}
@@ -608,8 +663,8 @@ export function fetchContainerStats(
import type { StaticSite, StaticSiteSecret, FolderEntry, GitProvider, RepoInfo } from './types';
export function listStaticSites(): Promise<StaticSite[]> {
return get<StaticSite[]>('/api/sites');
export function listStaticSites(signal?: AbortSignal): Promise<StaticSite[]> {
return get<StaticSite[]>('/api/sites', signal);
}
export function getStaticSite(id: string): Promise<StaticSite> {
@@ -710,4 +765,10 @@ export function deleteStaticSiteSecret(
return del<{ deleted: string }>(`/api/sites/${siteId}/secrets/${secretId}`);
}
export function getStaticSiteStorage(
siteId: string
): Promise<import('./types').StaticSiteStorageUsage> {
return get<import('./types').StaticSiteStorageUsage>(`/api/sites/${siteId}/storage`);
}
export { ApiError };
+11 -16
View File
@@ -18,24 +18,19 @@
let error = $state(false);
$effect(() => {
let cancelled = false;
let inflight = false;
let controller = new AbortController();
async function load() {
if (inflight) return; // Skip if previous request still pending.
inflight = true;
// Abort any previous in-flight request before starting a new one.
controller.abort();
controller = new AbortController();
try {
const result = await api.fetchContainerStats(projectId, stageId, instanceId);
if (!cancelled) {
stats = result;
error = false;
}
} catch {
if (!cancelled) {
error = true;
}
} finally {
inflight = false;
const result = await api.fetchContainerStats(projectId, stageId, instanceId, controller.signal);
stats = result;
error = false;
} catch (e) {
if (e instanceof DOMException && e.name === 'AbortError') return;
error = true;
}
}
@@ -45,7 +40,7 @@
const interval = setInterval(load, 30_000);
return () => {
cancelled = true;
controller.abort();
clearInterval(interval);
};
});
+2 -21
View File
@@ -8,12 +8,9 @@
import LocaleSwitcher from '$lib/components/LocaleSwitcher.svelte';
import { IconDashboard, IconProjects, IconDeploy, IconEvents, IconWifi, IconSettings, IconMenu, IconX, IconLogout, IconGlobe } from '$lib/components/icons';
import { goto } from '$app/navigation';
import { connectGlobalEvents, type SSEConnection } from '$lib/sse';
import { instanceStatusStore } from '$lib/stores/instance-status';
import { resolvedTheme, applyTheme } from '$lib/stores/theme';
import { exchangeOidcToken, setAuthToken, clearAuth, isAuthenticated } from '$lib/auth';
import { logout as apiLogout, getHealth } from '$lib/api';
import { publishEventLog } from '$lib/stores/event-log-bus';
import type { DockerHealth, ProxyHealth } from '$lib/types';
import { t } from '$lib/i18n';
@@ -38,7 +35,6 @@
return pathname.startsWith(href);
}
let sseConnection: SSEConnection | null = null;
let sidebarOpen = $state(false);
let dockerHealth = $state<DockerHealth | null>(null);
let proxyHealth = $state<ProxyHealth | null>(null);
@@ -85,26 +81,13 @@
}
});
// Start SSE and health polling when authenticated.
// Start health polling when authenticated.
// Uses $effect to react to route changes (e.g., after login navigation).
$effect(() => {
void $page.url.pathname;
if (!isAuthenticated() || sseConnection) return;
if (!isAuthenticated() || healthInterval) return;
sseConnection = connectGlobalEvents({
onInstanceStatus(payload) {
instanceStatusStore.update(payload);
},
onDeployStatus(payload) {
instanceStatusStore.notifyDeploy(payload);
},
onEventLog(payload) {
publishEventLog(payload);
}
});
// Poll Docker health every 30s.
async function checkHealth() {
try {
const h = await getHealth();
@@ -121,8 +104,6 @@
});
onDestroy(() => {
sseConnection?.close();
sseConnection = null;
if (healthInterval) clearInterval(healthInterval);
});
</script>
+114 -19
View File
@@ -1,11 +1,11 @@
<script lang="ts">
import type { Project, Instance, StaleContainer } from '$lib/types';
import type { Project, Instance, StaleContainer, StaticSite } from '$lib/types';
import * as api from '$lib/api';
import ProjectCard from '$lib/components/ProjectCard.svelte';
import SkeletonCard from '$lib/components/SkeletonCard.svelte';
import EmptyState from '$lib/components/EmptyState.svelte';
import SystemHealthCard from '$lib/components/SystemHealthCard.svelte';
import { IconDeploy, IconBox, IconServer, IconAlert, IconClock } from '$lib/components/icons';
import { IconDeploy, IconBox, IconServer, IconAlert, IconClock, IconGlobe } from '$lib/components/icons';
import { t } from '$lib/i18n';
let projects = $state<Project[]>([]);
@@ -14,46 +14,58 @@
let unusedImagesMB = $state(0);
let unusedImagesCount = $state(0);
let unusedImagesExceeded = $state(false);
let sites = $state<StaticSite[]>([]);
let loading = $state(true);
let error = $state('');
let loadController: AbortController | null = null;
async function loadDashboard() {
loadController?.abort();
const controller = new AbortController();
loadController = controller;
const signal = controller.signal;
loading = true;
error = '';
try {
projects = await api.listProjects();
projects = await api.listProjects(signal);
const detailPromises = projects.map(async (p) => {
// Fetch project details sequentially to avoid exhausting
// browser connection pool (HTTP/1.1 allows only 6 per host).
const results: { projectId: string; instances: Instance[] }[] = [];
for (const p of projects) {
try {
const detail = await api.getProject(p.id);
const detail = await api.getProject(p.id, signal);
const stages = detail.stages ?? [];
const stageInstances = await Promise.all(
stages.map((s) => api.listInstances(p.id, s.id))
);
return { projectId: p.id, instances: stageInstances.flat() };
} catch {
return { projectId: p.id, instances: [] };
const stageInstances: Instance[][] = [];
for (const s of stages) {
stageInstances.push(await api.listInstances(p.id, s.id, signal));
}
results.push({ projectId: p.id, instances: stageInstances.flat() });
} catch (e) {
if (e instanceof DOMException && e.name === 'AbortError') throw e;
results.push({ projectId: p.id, instances: [] });
}
});
}
const [results, staleResult] = await Promise.all([
Promise.all(detailPromises),
api.fetchStaleContainers().catch(() => [] as StaleContainer[])
]);
const mapped: Record<string, Instance[]> = {};
for (const r of results) {
mapped[r.projectId] = r.instances;
}
instancesByProject = mapped;
staleContainers = staleResult;
staleContainers = await api.fetchStaleContainers(signal).catch(() => [] as StaleContainer[]);
sites = await api.listStaticSites(signal).catch(() => [] as StaticSite[]);
try {
const imgStats = await api.getUnusedImageStats();
const imgStats = await api.getUnusedImageStats(signal);
unusedImagesMB = imgStats.total_size_mb;
unusedImagesCount = imgStats.count;
unusedImagesExceeded = imgStats.exceeded;
} catch { /* non-critical */ }
} catch (e) {
if (e instanceof DOMException && e.name === 'AbortError') return;
error = e instanceof Error ? e.message : $t('dashboard.loadFailed');
} finally {
loading = false;
@@ -62,6 +74,7 @@
$effect(() => {
loadDashboard();
return () => { loadController?.abort(); };
});
const totalProjects = $derived(projects.length);
@@ -76,6 +89,24 @@
.filter((i) => i.status === 'failed').length
);
const totalStale = $derived(staleContainers.length);
const totalSites = $derived(sites.length);
const deployedSites = $derived(sites.filter((s) => s.status === 'deployed').length);
const failedSitesCount = $derived(sites.filter((s) => s.status === 'failed').length);
function siteStatusBadge(status: string): { text: string; cls: string } {
switch (status) {
case 'deployed':
return { text: 'Deployed', cls: 'bg-emerald-100 text-emerald-700 dark:bg-emerald-900/30 dark:text-emerald-400' };
case 'syncing':
return { text: 'Syncing', cls: 'bg-blue-100 text-blue-700 dark:bg-blue-900/30 dark:text-blue-400' };
case 'failed':
return { text: 'Failed', cls: 'bg-red-100 text-red-700 dark:bg-red-900/30 dark:text-red-400' };
case 'stopped':
return { text: 'Stopped', cls: 'bg-gray-100 text-gray-600 dark:bg-gray-800 dark:text-gray-400' };
default:
return { text: 'Idle', cls: 'bg-gray-100 text-gray-600 dark:bg-gray-800 dark:text-gray-400' };
}
}
</script>
<svelte:head>
@@ -96,7 +127,7 @@
</div>
<!-- Stats cards -->
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-4">
<div class="grid grid-cols-1 gap-4 sm:grid-cols-2 lg:grid-cols-5">
<div class="flex items-center gap-4 rounded-xl border border-[var(--border-primary)] bg-[var(--surface-card)] p-5 shadow-[var(--shadow-sm)]">
<div class="flex h-12 w-12 items-center justify-center rounded-xl bg-[var(--color-brand-50)] text-[var(--color-brand-600)]">
<IconBox size={24} />
@@ -133,6 +164,23 @@
<p class="mt-0.5 text-2xl font-bold {totalStale > 0 ? 'text-amber-600' : 'text-[var(--text-primary)]'}">{totalStale}</p>
</div>
</a>
<a href="/sites" class="flex items-center gap-4 rounded-xl border border-[var(--border-primary)] bg-[var(--surface-card)] p-5 shadow-[var(--shadow-sm)] transition-colors hover:bg-[var(--surface-card-hover)]">
<div class="flex h-12 w-12 items-center justify-center rounded-xl {totalSites > 0 ? 'bg-[var(--color-brand-50)] text-[var(--color-brand-600)]' : 'bg-gray-50 text-gray-400'}">
<IconGlobe size={24} />
</div>
<div>
<p class="text-sm text-[var(--text-secondary)]">{$t('dashboard.totalSites')}</p>
<p class="mt-0.5 text-2xl font-bold text-[var(--text-primary)]">
{totalSites}
{#if deployedSites > 0}
<span class="text-sm font-medium text-emerald-600">{deployedSites} {$t('dashboard.deployedSites')}</span>
{/if}
{#if failedSitesCount > 0}
<span class="text-sm font-medium text-red-600">{failedSitesCount} {$t('dashboard.failedSites')}</span>
{/if}
</p>
</div>
</a>
</div>
<!-- Unused images warning -->
@@ -152,6 +200,53 @@
<!-- System health summary -->
<SystemHealthCard />
<!-- Static sites summary -->
{#if !loading}
<div>
<div class="flex items-center justify-between">
<h2 class="text-lg font-semibold text-[var(--text-primary)]">{$t('dashboard.staticSites')}</h2>
{#if sites.length > 0}
<a href="/sites" class="text-sm font-medium text-[var(--color-brand-600)] hover:text-[var(--color-brand-700)]">
{$t('dashboard.viewAllSites')} &rarr;
</a>
{/if}
</div>
{#if sites.length === 0}
<div class="mt-4">
<EmptyState
title={$t('dashboard.noSites')}
description={$t('dashboard.addFirstSite')}
actionLabel={$t('sites.title')}
actionHref="/sites"
icon="projects"
/>
</div>
{:else}
<div class="mt-4 grid grid-cols-1 gap-3 sm:grid-cols-2 lg:grid-cols-3">
{#each sites as site (site.id)}
{@const badge = siteStatusBadge(site.status)}
<a href="/sites/{site.id}" class="flex flex-col gap-2 rounded-xl border border-[var(--border-primary)] bg-[var(--surface-card)] p-4 shadow-[var(--shadow-sm)] transition-colors hover:bg-[var(--surface-card-hover)]">
<div class="flex items-center justify-between gap-2">
<span class="truncate font-medium text-[var(--text-primary)]">{site.name}</span>
<span class="inline-flex shrink-0 items-center rounded-full px-2 py-0.5 text-xs font-medium {badge.cls}">{badge.text}</span>
</div>
<div class="flex items-center gap-3 text-xs text-[var(--text-secondary)]">
<span class="truncate">{site.repo_owner}/{site.repo_name}</span>
{#if site.domain}
<span class="truncate text-[var(--color-brand-600)]">{site.domain}</span>
{/if}
</div>
{#if site.last_sync_at}
<p class="text-xs text-[var(--text-tertiary)]">{$t('sites.lastSync')}: {new Date(site.last_sync_at).toLocaleString()}</p>
{/if}
</a>
{/each}
</div>
{/if}
</div>
{/if}
<!-- Project cards -->
<div>
<h2 class="text-lg font-semibold text-[var(--text-primary)]">{$t('dashboard.projects')}</h2>
+9 -8
View File
@@ -8,8 +8,7 @@
import { fetchEventLog, fetchEventLogStats, clearAllEvents, deleteEvent } from '$lib/api';
import ConfirmDialog from '$lib/components/ConfirmDialog.svelte';
import { toasts } from '$lib/stores/toast';
import { subscribeEventLog } from '$lib/stores/event-log-bus';
import type { EventLogSSEPayload } from '$lib/sse';
import { connectGlobalEvents, type SSEConnection, type EventLogSSEPayload } from '$lib/sse';
import type { EventLogEntry, EventLogStats } from '$lib/types';
import EventLogEntryComponent from '$lib/components/EventLogEntry.svelte';
import EventLogFilter from '$lib/components/EventLogFilter.svelte';
@@ -36,7 +35,7 @@
const PAGE_SIZE = 50;
let offset = $state(0);
let unsubscribeEventLog: (() => void) | null = null;
let sseConnection: SSEConnection | null = null;
let listEl: HTMLDivElement | undefined = $state();
let showClearConfirm = $state(false);
@@ -199,15 +198,17 @@
loadEvents();
loadStats();
// Subscribe to event_log events from the global SSE connection (no duplicate connection).
unsubscribeEventLog = subscribeEventLog((payload: EventLogSSEPayload) => {
handleSSEEvent(payload);
// Open SSE connection only while this page is mounted.
sseConnection = connectGlobalEvents({
onEventLog(payload) {
handleSSEEvent(payload);
}
});
});
onDestroy(() => {
unsubscribeEventLog?.();
unsubscribeEventLog = null;
sseConnection?.close();
sseConnection = null;
});
</script>