feat(notify): HMAC-signed outgoing webhooks with per-tier secrets and test sender

Outgoing notifications were bare POSTs with no auth and no way to verify
they came from Tinyforge. They also went out from one global URL only,
even though stages had a notification_url field, and static-site sync
emitted no events at all.

Schema: add notification_url + notification_secret (lazy-generated) to
settings, projects, stages and static_sites. Migrations are additive.

Notifier: SendSigned computes HMAC-SHA256 over the exact body bytes and
sends X-Hub-Signature-256 (GitHub-compatible — receivers built for
GitHub/Gitea/Forgejo verify out of the box). Aux headers
X-Tinyforge-Event/Delivery/Timestamp/Tier are advisory and not signed.
Empty secret => unsigned send for back-compat.

Resolution: deploys fall through stage > project > settings, sites fall
through site > settings. The secret travels with the URL that sourced
it, so any tier can sign even when its parents are unsigned. Site sync
events now actually emit (site_sync_success / site_sync_failure).

API: 12 new endpoints — {GET secret, POST regenerate, POST disable,
POST test} for each of the 4 tiers. SendSyncForTest returns
status_code/latency_ms/signature_sent/delivery_id/response_snippet so
the UI surfaces receiver feedback inline.

UI: shared OutgoingWebhookPanel.svelte fits the existing card aesthetic.
Signing-state pill, secret reveal-on-demand, regenerate/disable behind
ConfirmDialog modals (not inline strips — too easy to misclick), send-
test result card with colour-coded status. Wired into Settings →
Integrations, project edit form, per-stage edit, and per-site detail.
EN + RU i18n.

Tests: round-trip (sender signs, receiver verifies), tampered-body and
wrong-secret rejection, unsigned-send omits header, send-test surfaces
4xx, concurrent fan-out via Drain. Resolver precedence locked for both
deploy and site paths.

Docs: docs/webhooks.md with header reference, verifier snippets in
Node/Python/Go, and a recipe for the service-to-notification-bridge
generic webhook provider.

internal/notify/notifier.go

@@ -3,17 +3,28 @@ package notify
 import (
 	"bytes"
 	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/http"
+	"net/url"
 	"sync"
 	"time"
+
+	"github.com/google/uuid"
 )
 
-// Event represents a deployment notification payload.
+// Event represents a deployment / site-sync notification payload.
+//
+// Field naming preserves backwards compatibility with the original
+// deploy_success/deploy_failure events; site events reuse Project for the
+// site name and leave Stage/ImageTag empty.
 type Event struct {
-	Type      string `json:"type"`      // "deploy_success" or "deploy_failure"
+	Type      string `json:"type"`      // deploy_success, deploy_failure, site_sync_success, site_sync_failure, test
 	Project   string `json:"project"`
 	Stage     string `json:"stage"`
 	ImageTag  string `json:"image_tag"`
@@ -23,8 +34,54 @@ type Event struct {
 	Timestamp string `json:"timestamp"`
 }
 
-// Notifier sends webhook notifications for deploy events.
-// Notifications are fire-and-forget — failures are logged but do not propagate.
+// Tier identifies which configuration layer supplied the URL+secret used for
+// a particular dispatch. Recorded in logs and the test-endpoint response so
+// operators can debug fall-through behaviour.
+type Tier string
+
+const (
+	TierSettings Tier = "settings"
+	TierProject  Tier = "project"
+	TierStage    Tier = "stage"
+	TierSite     Tier = "site"
+)
+
+// Header names for outgoing webhooks. The signature header name matches
+// GitHub/Gitea/Forgejo so receivers built for those providers (and the
+// service-to-notification-bridge generic webhook provider) verify out of the
+// box. The X-Tinyforge-* headers are advisory and not covered by the HMAC.
+const (
+	HeaderSignature = "X-Hub-Signature-256"
+	HeaderEvent     = "X-Tinyforge-Event"
+	HeaderDelivery  = "X-Tinyforge-Delivery"
+	HeaderTimestamp = "X-Tinyforge-Timestamp"
+	HeaderTier      = "X-Tinyforge-Tier"
+)
+
+// userAgent is reported on every outgoing webhook request so operators can
+// filter their access logs by source. Versioned tag is added later if/when
+// we wire build-time variables; for now a static identifier is enough.
+const userAgent = "Tinyforge-Webhook/1"
+
+// TestResult is what /api/.../notification-test returns to the UI: the
+// receiver's status code, latency, a short response preview, and whether a
+// signature was sent (so the operator can tell at a glance if signing is
+// configured for this tier).
+type TestResult struct {
+	URL              string `json:"url"`
+	Tier             Tier   `json:"tier"`
+	StatusCode       int    `json:"status_code"`
+	LatencyMs        int64  `json:"latency_ms"`
+	SignatureSent    bool   `json:"signature_sent"`
+	DeliveryID       string `json:"delivery_id"`
+	ResponseSnippet  string `json:"response_snippet"`
+	Error            string `json:"error,omitempty"`
+}
+
+// Notifier sends webhook notifications for deploy and site-sync events.
+// Notifications are fire-and-forget by default — failures are logged but do
+// not propagate. SendSyncForTest is the exception, used only by the manual
+// test endpoint.
 type Notifier struct {
 	httpClient *http.Client
 	wg         sync.WaitGroup
@@ -44,9 +101,20 @@ func (n *Notifier) Drain() {
 	n.wg.Wait()
 }
 
-// Send sends a notification event to the given webhook URL in a background goroutine.
-// It does not block the caller. Errors are logged, not returned.
+// Send dispatches an unsigned event to the given URL in the background.
+// Retained for callsites that don't yet have access to a signing secret;
+// new code should prefer SendSigned which records the resolution tier.
 func (n *Notifier) Send(webhookURL string, event Event) {
+	n.SendSigned(webhookURL, "", TierSettings, event)
+}
+
+// SendSigned dispatches an event, signing it with HMAC-SHA256 if secret is
+// non-empty. The signature is computed over the exact JSON bytes sent on the
+// wire (so receivers must verify the raw body, not a re-serialised copy).
+//
+// Empty secret => unsigned send (no X-Hub-Signature-256 header), preserving
+// the legacy behaviour for receivers that pre-date HMAC support.
+func (n *Notifier) SendSigned(webhookURL, secret string, tier Tier, event Event) {
 	if webhookURL == "" {
 		return
 	}
@@ -54,40 +122,154 @@ func (n *Notifier) Send(webhookURL string, event Event) {
 	if event.Timestamp == "" {
 		event.Timestamp = time.Now().UTC().Format(time.RFC3339)
 	}
+	delivery := uuid.NewString()
 
 	n.wg.Add(1)
 	go func() {
 		defer n.wg.Done()
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
-		if err := n.doSend(ctx, webhookURL, event); err != nil {
-			slog.Warn("notify: failed to send webhook", "url", webhookURL, "error", err)
+
+		_, err := n.doSend(ctx, webhookURL, secret, tier, delivery, event)
+		// URL host only — never log the secret or full URL with user-info.
+		host := safeHost(webhookURL)
+		if err != nil {
+			slog.Warn("notify: webhook send failed",
+				"tier", tier, "host", host, "delivery", delivery,
+				"event", event.Type, "signed", secret != "", "error", err)
+			return
 		}
+		slog.Info("notify: webhook dispatched",
+			"tier", tier, "host", host, "delivery", delivery,
+			"event", event.Type, "signed", secret != "")
 	}()
 }
 
-// doSend performs the actual HTTP POST to the webhook URL.
-func (n *Notifier) doSend(ctx context.Context, webhookURL string, event Event) error {
+// SendSyncForTest performs a synchronous, single-shot send for the "Send
+// test" UI button. Returns a TestResult describing what the receiver
+// answered with so the operator can confirm wiring without watching server
+// logs. Errors are reported via the Error field rather than the returned
+// error to keep the API ergonomic for the handler.
+func (n *Notifier) SendSyncForTest(ctx context.Context, webhookURL, secret string, tier Tier, event Event) TestResult {
+	if event.Timestamp == "" {
+		event.Timestamp = time.Now().UTC().Format(time.RFC3339)
+	}
+	delivery := uuid.NewString()
+	result := TestResult{
+		URL:           webhookURL,
+		Tier:          tier,
+		SignatureSent: secret != "",
+		DeliveryID:    delivery,
+	}
+
+	if webhookURL == "" {
+		result.Error = "no webhook URL configured for this tier"
+		return result
+	}
+
+	start := time.Now()
+	resp, err := n.doSend(ctx, webhookURL, secret, tier, delivery, event)
+	result.LatencyMs = time.Since(start).Milliseconds()
+	if err != nil {
+		result.Error = err.Error()
+		if resp != nil {
+			result.StatusCode = resp.StatusCode
+			result.ResponseSnippet = resp.BodyPreview
+		}
+		return result
+	}
+	result.StatusCode = resp.StatusCode
+	result.ResponseSnippet = resp.BodyPreview
+	return result
+}
+
+// sendResponse captures the small subset of the receiver's response we want
+// to surface back to the operator (status + a body preview). Distinct from
+// http.Response so callers don't accidentally hold an unread body.
+type sendResponse struct {
+	StatusCode  int
+	BodyPreview string
+}
+
+// doSend performs the HTTP POST, signs the body if a secret is configured,
+// and returns either a sendResponse (for the test path) or an error.
+//
+// The request body bytes are computed once so the HMAC signature matches
+// exactly what travels on the wire. Receivers MUST verify against the raw
+// body, not a re-serialised copy.
+func (n *Notifier) doSend(ctx context.Context, webhookURL, secret string, tier Tier, delivery string, event Event) (*sendResponse, error) {
 	body, err := json.Marshal(event)
 	if err != nil {
-		return fmt.Errorf("marshal notification: %w", err)
+		return nil, fmt.Errorf("marshal notification: %w", err)
 	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, webhookURL, bytes.NewReader(body))
 	if err != nil {
-		return fmt.Errorf("create notification request: %w", err)
+		return nil, fmt.Errorf("create notification request: %w", err)
 	}
 	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set(HeaderEvent, event.Type)
+	req.Header.Set(HeaderDelivery, delivery)
+	req.Header.Set(HeaderTimestamp, event.Timestamp)
+	req.Header.Set(HeaderTier, string(tier))
+	if secret != "" {
+		req.Header.Set(HeaderSignature, "sha256="+sign(secret, body))
+	}
 
 	resp, err := n.httpClient.Do(req)
 	if err != nil {
-		return fmt.Errorf("send notification: %w", err)
+		return nil, fmt.Errorf("send notification: %w", err)
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
-		return fmt.Errorf("notification webhook returned status %d", resp.StatusCode)
+	preview, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+	out := &sendResponse{
+		StatusCode:  resp.StatusCode,
+		BodyPreview: string(preview),
 	}
-
-	return nil
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return out, fmt.Errorf("notification webhook returned status %d", resp.StatusCode)
+	}
+	return out, nil
+}
+
+// sign returns the lowercase-hex HMAC-SHA256 of body using secret as the
+// key. The "sha256=" prefix is added by the caller to match GitHub's
+// X-Hub-Signature-256 wire format.
+func sign(secret string, body []byte) string {
+	mac := hmac.New(sha256.New, []byte(secret))
+	mac.Write(body)
+	return hex.EncodeToString(mac.Sum(nil))
+}
+
+// VerifySignature is the receiver-side counterpart to sign(). Exported so
+// our own tests (and any future incoming-webhook receiver in this repo) can
+// re-use the exact construction without duplicating the HMAC code.
+//
+// signatureHeader accepts either the raw hex digest or the GitHub-style
+// "sha256=<hex>" envelope.
+func VerifySignature(secret string, body []byte, signatureHeader string) bool {
+	if secret == "" || signatureHeader == "" {
+		return false
+	}
+	got := signatureHeader
+	if len(got) > 7 && got[:7] == "sha256=" {
+		got = got[7:]
+	}
+	want := sign(secret, body)
+	// hmac.Equal is the constant-time comparator; bytes.Equal would leak
+	// timing information about the first differing byte.
+	return hmac.Equal([]byte(got), []byte(want))
+}
+
+// safeHost extracts the host (and optional port) from a webhook URL for
+// logging. Returns the input unchanged if parsing fails so we never silently
+// swallow a malformed URL — operators see the failure mode either way.
+func safeHost(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil || u.Host == "" {
+		return "(unparseable)"
+	}
+	return u.Host
 }

internal/notify/notifier_test.go

@@ -0,0 +1,234 @@
+package notify_test
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+
+	"github.com/alexei/tinyforge/internal/notify"
+)
+
+// TestSignedRoundTrip is the canonical "the receiver can verify what we
+// sent" check. Sender signs the body with a secret; the test server reads
+// the raw body and the X-Hub-Signature-256 header and verifies via
+// VerifySignature. A regression here means receivers built against our
+// docs would silently reject real notifications.
+func TestSignedRoundTrip(t *testing.T) {
+	const secret = "super-secret-test-key-not-used-in-prod"
+
+	var receivedBody []byte
+	var receivedSig string
+	var receivedEvent string
+	var receivedDelivery string
+	var receivedTier string
+	var receivedTimestamp string
+	done := make(chan struct{})
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer close(done)
+		body, _ := io.ReadAll(r.Body)
+		receivedBody = body
+		receivedSig = r.Header.Get(notify.HeaderSignature)
+		receivedEvent = r.Header.Get(notify.HeaderEvent)
+		receivedDelivery = r.Header.Get(notify.HeaderDelivery)
+		receivedTier = r.Header.Get(notify.HeaderTier)
+		receivedTimestamp = r.Header.Get(notify.HeaderTimestamp)
+		w.WriteHeader(http.StatusOK)
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+	defer srv.Close()
+
+	n := notify.New()
+	n.SendSigned(srv.URL, secret, notify.TierStage, notify.Event{
+		Type:    "deploy_success",
+		Project: "demo",
+		Stage:   "prod",
+	})
+	n.Drain()
+	<-done
+
+	if !notify.VerifySignature(secret, receivedBody, receivedSig) {
+		t.Fatalf("receiver could not verify signature: header=%q body=%q", receivedSig, receivedBody)
+	}
+	if receivedEvent != "deploy_success" {
+		t.Errorf("event header = %q, want deploy_success", receivedEvent)
+	}
+	if receivedDelivery == "" {
+		t.Errorf("delivery ID header missing")
+	}
+	if receivedTier != string(notify.TierStage) {
+		t.Errorf("tier header = %q, want %q", receivedTier, notify.TierStage)
+	}
+	if receivedTimestamp == "" {
+		t.Errorf("timestamp header missing")
+	}
+
+	// Sanity: payload roundtrips through JSON unchanged.
+	var got notify.Event
+	if err := json.Unmarshal(receivedBody, &got); err != nil {
+		t.Fatalf("decode body: %v", err)
+	}
+	if got.Project != "demo" || got.Stage != "prod" {
+		t.Errorf("body fields lost in transit: %+v", got)
+	}
+}
+
+// TestUnsignedSendOmitsSignatureHeader covers the back-compat path: a
+// caller that hasn't yet generated a signing secret should still be able to
+// dispatch events, just without the signature header. Existing receivers
+// must not break when a Tinyforge instance upgrades but hasn't enabled
+// signing yet.
+func TestUnsignedSendOmitsSignatureHeader(t *testing.T) {
+	var sigHeader string
+	done := make(chan struct{})
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer close(done)
+		sigHeader = r.Header.Get(notify.HeaderSignature)
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	n := notify.New()
+	n.SendSigned(srv.URL, "", notify.TierSettings, notify.Event{Type: "test"})
+	n.Drain()
+	<-done
+
+	if sigHeader != "" {
+		t.Errorf("expected no signature header on unsigned send, got %q", sigHeader)
+	}
+}
+
+// TestVerifyRejectsTamperedBody is the negative half of the round-trip:
+// flipping a single byte in the signed body must fail verification.
+// Catches accidental MAC truncation / wrong hash family / non-constant-time
+// compares (the last only weakly, but the round-trip already guards
+// correctness; this just locks the contract).
+func TestVerifyRejectsTamperedBody(t *testing.T) {
+	const secret = "abc"
+	body := []byte(`{"type":"deploy_success"}`)
+	sig := "sha256=" + hexEncode(hmacSha256(secret, body))
+
+	if !notify.VerifySignature(secret, body, sig) {
+		t.Fatalf("control: legit signature failed to verify")
+	}
+
+	tampered := append([]byte(nil), body...)
+	tampered[1] = 'X' // flip one byte
+	if notify.VerifySignature(secret, tampered, sig) {
+		t.Errorf("verifier accepted tampered body — signature scheme is broken")
+	}
+
+	if notify.VerifySignature("wrong-secret", body, sig) {
+		t.Errorf("verifier accepted wrong secret")
+	}
+	if notify.VerifySignature(secret, body, "") {
+		t.Errorf("verifier accepted empty signature header")
+	}
+	if notify.VerifySignature("", body, sig) {
+		t.Errorf("verifier accepted empty secret")
+	}
+}
+
+// TestSendSyncForTestReturnsReceiverStatus is the "send test" UI button
+// contract: when the receiver returns a non-2xx status, we must surface
+// both the status code and the body preview rather than swallowing them.
+// Operators rely on this to debug mis-pointed receivers.
+func TestSendSyncForTestReturnsReceiverStatus(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusForbidden)
+		_, _ = w.Write([]byte("invalid signature"))
+	}))
+	defer srv.Close()
+
+	n := notify.New()
+	res := n.SendSyncForTest(context.Background(), srv.URL, "secret", notify.TierProject, notify.Event{Type: "test"})
+
+	if res.StatusCode != http.StatusForbidden {
+		t.Errorf("status_code = %d, want 403", res.StatusCode)
+	}
+	if res.ResponseSnippet != "invalid signature" {
+		t.Errorf("response_snippet = %q, want 'invalid signature'", res.ResponseSnippet)
+	}
+	if !res.SignatureSent {
+		t.Errorf("signature_sent should be true when secret is provided")
+	}
+	if res.Tier != notify.TierProject {
+		t.Errorf("tier = %q, want project", res.Tier)
+	}
+	if res.Error == "" {
+		t.Errorf("Error field should be set on 4xx response")
+	}
+}
+
+// TestSendSyncForTestEmptyURL is the guard for the test endpoint when no
+// URL is configured at any tier. The handler relies on Error being non-empty
+// to render the "no URL configured" message, so this contract must hold.
+func TestSendSyncForTestEmptyURL(t *testing.T) {
+	n := notify.New()
+	res := n.SendSyncForTest(context.Background(), "", "secret", notify.TierSettings, notify.Event{Type: "test"})
+	if res.Error == "" {
+		t.Errorf("Error field should be set when URL is empty")
+	}
+	if res.StatusCode != 0 {
+		t.Errorf("StatusCode should remain 0 when no request was made, got %d", res.StatusCode)
+	}
+}
+
+// TestConcurrentSendsAllArrive guards the WaitGroup contract on Drain — a
+// regression where Drain returns before in-flight goroutines complete would
+// drop notifications during graceful shutdown.
+func TestConcurrentSendsAllArrive(t *testing.T) {
+	const fanout = 20
+	var (
+		mu       sync.Mutex
+		received int
+	)
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		received++
+		mu.Unlock()
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer srv.Close()
+
+	n := notify.New()
+	for i := 0; i < fanout; i++ {
+		n.SendSigned(srv.URL, "key", notify.TierSettings, notify.Event{Type: "test"})
+	}
+	n.Drain()
+
+	mu.Lock()
+	defer mu.Unlock()
+	if received != fanout {
+		t.Errorf("received %d sends, want %d", received, fanout)
+	}
+}
+
+// --- helpers ----------------------------------------------------------
+
+// hmacSha256 + hexEncode duplicate the production sign() body so the
+// negative tests don't depend on the un-exported helper. If the exported
+// VerifySignature contract changes, this is the canary.
+func hmacSha256(secret string, body []byte) []byte {
+	h := hmac.New(sha256.New, []byte(secret))
+	h.Write(body)
+	return h.Sum(nil)
+}
+
+func hexEncode(b []byte) string {
+	const hexdigits = "0123456789abcdef"
+	out := make([]byte, len(b)*2)
+	for i, x := range b {
+		out[i*2] = hexdigits[x>>4]
+		out[i*2+1] = hexdigits[x&0x0f]
+	}
+	return string(out)
+}