feat(observability): event triggers + log scanner backend
Two paired backends sharing the events.Bus seam:
Event triggers (consumer-side):
- internal/store/event_triggers.go — CRUD with action_secret
redaction on read (placeholder echo treated as "no change" on
PATCH so secrets aren't accidentally wiped).
- internal/events/dispatcher.go — bus subscriber, AND-composed
filters (severity CSV, source CSV, message regex with memoized
compile cache). Structural loop-prevention: never writes to
event_log. Sends via notifier.SendPayload.
- internal/notify: SendPayload + SendSyncForTestPayload methods,
TierEventTrigger constant, doSendRaw shared with the legacy
Event-shaped path.
- internal/api/event_triggers.go — admin-gated CRUD + /test
sending the real TriggerWebhookPayload shape. SSRF guard
rejects loopback / link-local / unspecified targets. PATCH
uses pointer-typed DTO for partial updates.
Log scanner (producer-side):
- internal/logscanner/ — engine (per-rule cooldown +
per-container token bucket, atomic drop counters), tail
(multiplexed docker frame demuxer with TTY fallback + 16 MiB
payload cap + 1 MiB reassembly cap + RFC3339Nano-validated
timestamp strip + UTF-8-safe message truncation), manager
(5s container polling, atomic.Pointer[Snapshot] hot-reload,
HitEmitter writes event_log + publishes EventLog so the
trigger dispatcher picks them up immediately).
- internal/docker/container.go — ContainerLogsOpts exposes
stream selection for stderr-only / stdout-only rules.
- internal/store: log_scan_rules table + CRUD with
EffectiveLogScanRules resolver (globals minus per-workload
overrides plus workload-only additions). Transactional
cascade-delete of overrides when a global rule is removed.
- internal/api/log_scan_rules.go — admin-gated CRUD + /test
(sample_line → matched/captures) + /stats (drop counters +
active tail count + last-snapshot compile errors) +
GET /api/workloads/{id}/effective-rules.
cmd/server/main.go wires both subsystems next to the existing
RegisterPersistentLogger. Coverage spans engine cooldown / bucket
counter tests, snapshot effective-set semantics, manager compile-
error capture, dispatcher matching, store validation +
cascade-delete, API URL validator + secret redaction.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -40,10 +40,11 @@ type Event struct {
|
||||
type Tier string
|
||||
|
||||
const (
|
||||
TierSettings Tier = "settings"
|
||||
TierProject Tier = "project"
|
||||
TierStage Tier = "stage"
|
||||
TierSite Tier = "site"
|
||||
TierSettings Tier = "settings"
|
||||
TierProject Tier = "project"
|
||||
TierStage Tier = "stage"
|
||||
TierSite Tier = "site"
|
||||
TierEventTrigger Tier = "event_trigger"
|
||||
)
|
||||
|
||||
// Header names for outgoing webhooks. The signature header name matches
|
||||
@@ -145,6 +146,43 @@ func (n *Notifier) SendSigned(webhookURL, secret string, tier Tier, event Event)
|
||||
}()
|
||||
}
|
||||
|
||||
// SendPayload dispatches an arbitrary JSON payload to the given URL,
|
||||
// signed with HMAC-SHA256 when secret is non-empty. Used by the
|
||||
// event-trigger dispatcher: event-log → trigger filter → webhook
|
||||
// delivery. The eventType travels in the X-Tinyforge-Event header so
|
||||
// receivers can route by it without parsing the body.
|
||||
//
|
||||
// Fire-and-forget. Failures are logged at warn but never propagate;
|
||||
// trigger reliability is observed via webhook_deliveries (audit trail)
|
||||
// and the dispatcher remaining bus-driven means delivery hiccups
|
||||
// cannot back-pressure event publishing.
|
||||
func (n *Notifier) SendPayload(webhookURL, secret, eventType string, payload any) {
|
||||
if webhookURL == "" {
|
||||
return
|
||||
}
|
||||
delivery := uuid.NewString()
|
||||
timestamp := time.Now().UTC().Format(time.RFC3339)
|
||||
|
||||
n.wg.Add(1)
|
||||
go func() {
|
||||
defer n.wg.Done()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := n.doSendRaw(ctx, webhookURL, secret, TierEventTrigger, delivery, eventType, timestamp, payload)
|
||||
host := safeHost(webhookURL)
|
||||
if err != nil {
|
||||
slog.Warn("notify: trigger webhook send failed",
|
||||
"tier", TierEventTrigger, "host", host, "delivery", delivery,
|
||||
"event", eventType, "signed", secret != "", "error", err)
|
||||
return
|
||||
}
|
||||
slog.Info("notify: trigger webhook dispatched",
|
||||
"tier", TierEventTrigger, "host", host, "delivery", delivery,
|
||||
"event", eventType, "signed", secret != "")
|
||||
}()
|
||||
}
|
||||
|
||||
// SendSyncForTest performs a synchronous, single-shot send for the "Send
|
||||
// test" UI button. Returns a TestResult describing what the receiver
|
||||
// answered with so the operator can confirm wiring without watching server
|
||||
@@ -183,6 +221,42 @@ func (n *Notifier) SendSyncForTest(ctx context.Context, webhookURL, secret strin
|
||||
return result
|
||||
}
|
||||
|
||||
// SendSyncForTestPayload is the arbitrary-payload counterpart to
|
||||
// SendSyncForTest. Returns the same TestResult shape but sends an
|
||||
// arbitrary payload + event-type pair through the shared HTTP+HMAC
|
||||
// core. Used by the event-trigger /test endpoint so the operator's
|
||||
// receiver sees the same envelope shape it will receive during normal
|
||||
// dispatch — verifying a different payload would defeat the test's
|
||||
// purpose.
|
||||
func (n *Notifier) SendSyncForTestPayload(ctx context.Context, webhookURL, secret string, tier Tier, eventType string, payload any) TestResult {
|
||||
delivery := uuid.NewString()
|
||||
timestamp := time.Now().UTC().Format(time.RFC3339)
|
||||
result := TestResult{
|
||||
URL: webhookURL,
|
||||
Tier: tier,
|
||||
SignatureSent: secret != "",
|
||||
DeliveryID: delivery,
|
||||
}
|
||||
if webhookURL == "" {
|
||||
result.Error = "no webhook URL configured for this tier"
|
||||
return result
|
||||
}
|
||||
start := time.Now()
|
||||
resp, err := n.doSendRaw(ctx, webhookURL, secret, tier, delivery, eventType, timestamp, payload)
|
||||
result.LatencyMs = time.Since(start).Milliseconds()
|
||||
if err != nil {
|
||||
result.Error = err.Error()
|
||||
if resp != nil {
|
||||
result.StatusCode = resp.StatusCode
|
||||
result.ResponseSnippet = resp.BodyPreview
|
||||
}
|
||||
return result
|
||||
}
|
||||
result.StatusCode = resp.StatusCode
|
||||
result.ResponseSnippet = resp.BodyPreview
|
||||
return result
|
||||
}
|
||||
|
||||
// sendResponse captures the small subset of the receiver's response we want
|
||||
// to surface back to the operator (status + a body preview). Distinct from
|
||||
// http.Response so callers don't accidentally hold an unread body.
|
||||
@@ -198,7 +272,16 @@ type sendResponse struct {
|
||||
// exactly what travels on the wire. Receivers MUST verify against the raw
|
||||
// body, not a re-serialised copy.
|
||||
func (n *Notifier) doSend(ctx context.Context, webhookURL, secret string, tier Tier, delivery string, event Event) (*sendResponse, error) {
|
||||
body, err := json.Marshal(event)
|
||||
return n.doSendRaw(ctx, webhookURL, secret, tier, delivery, event.Type, event.Timestamp, event)
|
||||
}
|
||||
|
||||
// doSendRaw is the shared HTTP+HMAC core. It serializes any payload to
|
||||
// JSON, signs the resulting bytes (if a secret is configured) and
|
||||
// dispatches with the same Tinyforge headers as the legacy deploy-event
|
||||
// path. Separated out so SendPayload can reuse it without forcing the
|
||||
// caller to fit into the Event shape.
|
||||
func (n *Notifier) doSendRaw(ctx context.Context, webhookURL, secret string, tier Tier, delivery, eventType, timestamp string, payload any) (*sendResponse, error) {
|
||||
body, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal notification: %w", err)
|
||||
}
|
||||
@@ -209,9 +292,9 @@ func (n *Notifier) doSend(ctx context.Context, webhookURL, secret string, tier T
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", userAgent)
|
||||
req.Header.Set(HeaderEvent, event.Type)
|
||||
req.Header.Set(HeaderEvent, eventType)
|
||||
req.Header.Set(HeaderDelivery, delivery)
|
||||
req.Header.Set(HeaderTimestamp, event.Timestamp)
|
||||
req.Header.Set(HeaderTimestamp, timestamp)
|
||||
req.Header.Set(HeaderTier, string(tier))
|
||||
if secret != "" {
|
||||
req.Header.Set(HeaderSignature, "sha256="+sign(secret, body))
|
||||
|
||||
Reference in New Issue
Block a user