Name alert and MFA audit events after what they report

docs/src/content/docs/deployment/terraform/s3-worm-storage.mdx

@@ -515,10 +515,11 @@ fails and alerts.
   `AUDIT_HMAC_SECRET`, which only `DEV_MODE` allows, it doesn't run.
 - **Alerts.** Every failing run (an integrity failure, or a run that couldn't read the database or the bucket) is
   sent to Sentry, where runs that keep failing group into one issue. The gateway also records an
-  `audit_log.verify` audit event with status `failed` or `error` and the first failures, which goes to every Slack
-  channel whose action patterns match it (the default `#security` channel's do): on the first failing run, when a
-  run finds more failures than the last alert reported, and once a day while it keeps failing. The first passing
-  run after that records one with status `success`. Warnings don't alert. Findings that can't be fixed, such as a
+  `audit_log.verification_failed` audit event with status `failed` or `error` and the first failures, which goes to
+  every Slack channel whose action patterns match it (the default `#security` channel's `audit_log.verification_*`
+  does): on the first failing run, when a run finds more failures than the last alert reported, and once a day while
+  it keeps failing. The first passing run after that records `audit_log.verification_recovered`. A passing run
+  otherwise records nothing, and warnings don't alert. Findings that can't be fixed, such as a
   locked anchor that an earlier version wrote twice, keep verification failing until the anchor's retention ends.
   To be alerted when runs stop, set `AUDIT_VERIFY_HEARTBEAT_URL` to a heartbeat monitor's URL; the gateway requests
   it after every run (see [Heartbeats](/operations/monitoring/#heartbeats)).

docs/src/content/docs/integrations/slack.mdx

@@ -96,10 +96,11 @@ The gateway creates two default channel configs:
 - `login.complete`
 - `login.*_failed`
 - `mfa_method.*`
+- `mfa.verify_failed` (a failed MFA verification, in the web UI or for a CLI login)
 - `user.update_roles`
 - `api_token.*`
 - `env.*_protected`
-- `audit_log.verify` (the [hourly audit verification](/deployment/terraform/s3-worm-storage/#hourly-verification-in-the-gateway) failing, and passing again)
+- `audit_log.verification_*` (the [hourly audit verification](/deployment/terraform/s3-worm-storage/#hourly-verification-in-the-gateway) failing, and passing again after failing; a passing run posts nothing)
 
 **#infrastructure**
 - `deploy_approval_request.*`

docs/src/content/docs/operations/upgrades.mdx

@@ -332,8 +332,10 @@ itself every hour and alerts when they fail.
    From then on the gateway runs the same checks every hour, sends failures to Sentry and Slack, and shows the
    latest result on the **Audit Logs** page
    ([Hourly verification in the gateway](/deployment/terraform/s3-worm-storage/#hourly-verification-in-the-gateway)).
-   Slack integrations connected from this release on send the alerts (action `audit_log.verify`) to `#security`;
-   for an existing integration, add `audit_log.verify` to a channel's action patterns on the **Integrations** page.
+   Slack integrations connected from this release on send the alerts (actions `audit_log.verification_failed` and
+   `audit_log.verification_recovered`) to `#security`; for an existing integration, add `audit_log.verification_*`
+   to a channel's action patterns on the **Integrations** page. The same goes for failed MFA verifications
+   (`mfa.verify_failed`).
    Verification also fails when no anchor has been written for 3 hours.
 6. **Expect findings in anchors from earlier versions.** Anchors written before this release have no `format`.
    Their `prev_anchor_hash` stopped naming the previous anchor once the bucket held 1000 anchors; those are

internal/gateway/CLAUDE.md

@@ -161,8 +161,9 @@ Key files:
   with the command). Full check on start, while failing and daily; otherwise the events since the last passing run.
   What it has verified and alerted is in memory only (a database writer must not be able to skip checks or silence
   alerts); `audit_verification_status` is display-only (Audit Logs page). Failures go to Sentry every run and to
-  Slack as `audit_log.verify` audit events (first failure, more failures, daily reminder, recovery). Every run
-  pings `AUDIT_VERIFY_HEARTBEAT_URL`, so a heartbeat monitor alerts when runs stop.
+  Slack as audit events: `audit_log.verification_failed` (first failure, more failures, daily reminder) and
+  `audit_log.verification_recovered` (passing again). A passing run records no event. Every run pings
+  `AUDIT_VERIFY_HEARTBEAT_URL`, so a heartbeat monitor alerts when runs stop.
 - `heartbeat/`, `jobs/uptime/` - Heartbeat pings (https URLs from env vars, never logged: they carry the monitor's
   key). `UPTIME_HEARTBEAT_URL` is pinged on start and every `UPTIME_HEARTBEAT_INTERVAL_MINUTES` while the health
   check would pass.

internal/gateway/audit/constants.go

@@ -19,10 +19,10 @@ var (
 	ActionScopeDeployApprovalRequest = rbac.ResourceDeployApprovalRequest.String()
 	ActionScopeLogin                 = "login"
 	ActionScopeLogout                = "logout"
+	ActionScopeMFA                   = "mfa"
 	ActionScopeMFABackupCodes        = rbac.ResourceMFABackupCodes.String()
 	ActionScopeMFAMethod             = rbac.ResourceMFAMethod.String()
 	ActionScopeMFAPreferences        = rbac.ResourceMFAPreferences.String()
-	ActionScopeMFAVerification       = rbac.ResourceMFAVerification.String()
 	ActionScopeRateLimit             = "rate_limit"
 	ActionScopeRelease               = rbac.ResourceRelease.String()
 	ActionScopeSuspiciousActivity    = "suspicious_activity"
@@ -32,6 +32,18 @@ var (
 // ActionSessionCreate names the audit entries of a login creating a session (web or CLI).
 const ActionSessionCreate = "session.create"
 
+// MFA verification (rbac's auditActions names its routes with the same strings).
+const (
+	// ActionMFAVerify names every MFA verification, in the web UI or for a CLI login: its attempt, and its result
+	// with status success or failed.
+	ActionMFAVerify = "mfa.verify"
+	// ActionMFAVerifyFailed is the alert for a failed MFA verification (or a wrong code while enrolling), recorded
+	// besides the result: it emails the user and goes to the Slack channels whose patterns match it.
+	ActionMFAVerifyFailed = "mfa.verify_failed"
+	// ActionMFAChallenge names starting a WebAuthn verification, which issues its challenge.
+	ActionMFAChallenge = "mfa.challenge"
+)
+
 // Action verb constants (second part of action, e.g., "start" in "login.start")
 const (
 	ActionVerbBackupCodeRevealed = "backup_code_revealed"
@@ -52,7 +64,9 @@ const (
 	ActionVerbUpdateRoles        = "update_roles"
 	ActionVerbUserNotAuthorized  = "user_not_authorized"
 	ActionVerbValidate           = "validate"
-	ActionVerbVerify             = "verify"
+	// The scheduled audit log verification failing, and passing again after failing
+	ActionVerbVerificationFailed    = "verification_failed"
+	ActionVerbVerificationRecovered = "verification_recovered"
 )
 
 // Status constants

internal/gateway/audit/mfa_actions_test.go

@@ -0,0 +1,29 @@
+package audit
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/DocSpring/rack-gateway/internal/gateway/rbac"
+)
+
+// rbac names the MFA verification routes with these actions, as strings (it can't import them), and the CLI login
+// records its MFA step with ActionMFAVerify: one name for every MFA verification.
+func TestMFAVerificationRoutesUseTheMFAActions(t *testing.T) {
+	cases := []struct {
+		method, pattern, action string
+	}{
+		{"POST", "/api/v1/auth/mfa/verify", ActionMFAVerify},
+		{"POST", "/api/v1/auth/mfa/webauthn/assertion/verify", ActionMFAVerify},
+		{"POST", "/api/v1/auth/mfa/webauthn/assertion/start", ActionMFAChallenge},
+	}
+	for _, tc := range cases {
+		spec, ok := rbac.LookupHTTPRoute(tc.method, tc.pattern)
+		require.True(t, ok, "%s %s", tc.method, tc.pattern)
+		name := spec.AuditName(nil)
+		assert.Equal(t, tc.action, name.Action, "%s %s", tc.method, tc.pattern)
+		assert.Equal(t, ActionScopeMFA, name.ResourceType, "%s %s", tc.method, tc.pattern)
+	}
+}

internal/gateway/handlers/auth_cli.go

@@ -313,24 +313,21 @@ func (h *AuthHandler) CLILoginMFASubmit(c *gin.Context) {
 		return
 	}
 
-	if !h.recordAuthAttempt(c, cliLoginMFAAction, "cli", userRecord.Email, userRecord.Name) {
-		c.JSON(http.StatusServiceUnavailable, gin.H{"error": audit.RefusalMessage(cliLoginMFAAction)})
+	if !h.recordAuthAttempt(c, audit.ActionMFAVerify, "cli", userRecord.Email, userRecord.Name) {
+		c.JSON(http.StatusServiceUnavailable, gin.H{"error": audit.RefusalMessage(audit.ActionMFAVerify)})
 		return
 	}
 	verified := h.verifyCLILoginMFA(c, userRecord, parsed)
 	status := audit.StatusSuccess
 	if !verified {
 		status = audit.StatusFailed
 	}
-	h.recordAuthResult(c, cliLoginMFAAction, "cli", userRecord.Email, userRecord.Name, status)
+	h.recordAuthResult(c, audit.ActionMFAVerify, "cli", userRecord.Email, userRecord.Name, status)
 	if verified {
 		c.JSON(http.StatusOK, gin.H{"redirect": cliReturnRoute(parsed.state)})
 	}
 }
 
-// cliLoginMFAAction names the audit entries of a CLI login's MFA verification.
-const cliLoginMFAAction = "mfa.verify"
-
 // verifyCLILoginMFA verifies the MFA proof a CLI login submitted and marks the login (and the browser session it
 // came from) verified. It answers the request itself when verification fails, and reports whether it succeeded.
 func (h *AuthHandler) verifyCLILoginMFA(c *gin.Context, user *db.User, parsed cliMFASubmit) bool {

internal/gateway/handlers/integrations_slack.go

@@ -26,12 +26,13 @@ var defaultChannelActions = map[string]interface{}{
 			audit.BuildAction(audit.ActionScopeLogin, audit.ActionVerbComplete),
 			audit.ActionScopeLogin + ".*_failed", // Matches oauth_failed, user_not_authorized, etc.
 			audit.ActionScopeMFAMethod + ".*",    // MFA enrollment events
+			audit.ActionMFAVerifyFailed,          // Failed MFA verifications
 			audit.BuildAction(rbac.ResourceUser.String(), audit.ActionVerbUpdateRoles),
 			audit.ActionScopeAPIToken + ".*", // Matches all API token actions
 			// Protected env var changes (env.set_protected, env.unset_protected) and refused attempts to make them
 			rbac.ResourceEnv.String() + ".*_protected",
 			// The scheduled audit log verification failing, and passing again
-			jobverifyaudit.AlertAction,
+			jobverifyaudit.AlertActionPattern,
 		},
 	},
 	"infrastructure": map[string]interface{}{

internal/gateway/jobs/verifyaudit/alerts.go

@@ -14,9 +14,17 @@ import (
 // maxAlertedFailures is how many failure messages an alert lists. The summary on the Audit Logs page has them all.
 const maxAlertedFailures = 10
 
-// AlertAction is the audit action of verification alerts. A Slack channel gets them when one of its action
-// patterns matches it; new Slack integrations route it to their #security channel.
-var AlertAction = audit.BuildAction(audit.ActionScopeAuditLog, audit.ActionVerbVerify)
+// The audit actions of verification alerts. A passing run records neither. A Slack channel gets them when one of
+// its action patterns matches; AlertActionPattern matches both, and new Slack integrations route it to their
+// #security channel.
+var (
+	// FailureAction alerts that verification is failing: status failed, or error when the run couldn't verify.
+	FailureAction = audit.BuildAction(audit.ActionScopeAuditLog, audit.ActionVerbVerificationFailed)
+	// RecoveryAction says verification passes again after failing (status success).
+	RecoveryAction = audit.BuildAction(audit.ActionScopeAuditLog, audit.ActionVerbVerificationRecovered)
+	// AlertActionPattern matches FailureAction and RecoveryAction.
+	AlertActionPattern = audit.BuildAction(audit.ActionScopeAuditLog, "verification_*")
+)
 
 // sentryFingerprint groups failing runs into one Sentry issue per result, so runs that keep failing add events
 // to it instead of opening an issue every hour.
@@ -53,8 +61,8 @@ func (s *runState) decideAlert(previous *db.AuditVerificationStatus, o *outcome,
 	}
 }
 
-// sendAlert records the alert as an audit event, which reaches the Slack channels whose patterns match
-// AlertAction. It reports whether it was recorded; a failure alert that wasn't is sent again by the next run.
+// sendAlert records the alert as an audit event, which reaches the Slack channels whose patterns match it. It
+// reports whether it was recorded; a failure alert that wasn't is sent again by the next run.
 func (w *Worker) sendAlert(kind alertKind, o *outcome, since time.Time) bool {
 	if kind == noAlert {
 		return false
@@ -69,12 +77,14 @@ func (w *Worker) sendAlert(kind alertKind, o *outcome, since time.Time) bool {
 // alertEntry is the audit event for an alert: a failure (status failed, or error when the run couldn't verify)
 // with the first failures, or a recovery (status success).
 func alertEntry(kind alertKind, o *outcome, since time.Time) *db.AuditLog {
+	action := RecoveryAction
 	status := audit.StatusSuccess
 	details := map[string]interface{}{
 		"result":        o.result(),
 		"failing_since": since.UTC().Format(time.RFC3339),
 	}
 	if kind == failureAlert {
+		action = FailureAction
 		status = audit.StatusFailed
 		details["failure_count"] = o.failureCount()
 		if o.err != nil {
@@ -92,7 +102,7 @@ func alertEntry(kind alertKind, o *outcome, since time.Time) *db.AuditLog {
 		UserEmail:  audit.SystemActor,
 		UserName:   audit.SystemActor,
 		ActionType: audit.ActionTypeSecurity,
-		Action:     AlertAction,
+		Action:     action,
 		Details:    string(encoded),
 		Status:     status,
 	}

internal/gateway/jobs/verifyaudit/helpers_test.go

@@ -63,7 +63,7 @@ func newHarness(t *testing.T, store auditverify.AnchorStore) *harness {
 	h := &harness{t: t, database: dbtest.NewDatabase(t), slack: &queuedSlackEvents{}, now: firstRun}
 	_, err := h.database.CreateSlackIntegration("T123", "Test", "xoxb-test-token", "U123", "chat:write",
 		map[string]interface{}{"security": map[string]interface{}{
-			"id": "C123", "name": "#security", "actions": []string{AlertAction},
+			"id": "C123", "name": "#security", "actions": []string{AlertActionPattern},
 		}}, nil)
 	require.NoError(t, err)
 

internal/gateway/jobs/verifyaudit/worker_test.go

@@ -66,7 +66,7 @@ func TestRunAlertsOnFailureAndRecovery(t *testing.T) {
 	assert.Contains(t, h.sentryEvents[0].Message, "1 integrity failures")
 	alerts := h.slackAlerts()
 	require.Len(t, alerts, 1)
-	assert.Equal(t, "audit_log.verify", alerts[0].Action)
+	assert.Equal(t, "audit_log.verification_failed", alerts[0].Action)
 	assert.Equal(t, audit.StatusFailed, alerts[0].Status)
 	assert.Equal(t, audit.SystemActor, alerts[0].UserEmail)
 	assert.Contains(t, alerts[0].Details, fmt.Sprintf("event %d (id", tampered))
@@ -103,6 +103,8 @@ func TestRunAlertsOnFailureAndRecovery(t *testing.T) {
 	assert.Nil(t, status.FailingSince)
 	alerts = h.slackAlerts()
 	require.Len(t, alerts, 4)
+	assert.Equal(t, "audit_log.verification_failed", alerts[2].Action, "the daily reminder")
+	assert.Equal(t, "audit_log.verification_recovered", alerts[3].Action)
 	assert.Equal(t, audit.StatusSuccess, alerts[3].Status)
 	assert.Contains(t, alerts[3].Details, `"result":"passed"`)
 	assert.Len(t, h.sentryEvents, 4, "a passing run isn't sent to Sentry")
@@ -180,6 +182,7 @@ func TestRunAfterARestart(t *testing.T) {
 	alerts := h.slackAlerts()
 	require.Len(t, alerts, 3)
 	assert.Equal(t, audit.StatusSuccess, alerts[2].Status, "passing after a restart is alerted")
+	assert.Equal(t, "audit_log.verification_recovered", alerts[2].Action)
 }
 
 // Removing events from the end leaves a valid chain, but not one that holds the last event a run checked, even
@@ -214,6 +217,7 @@ func TestRunRecordsWhyItCouldNotVerify(t *testing.T) {
 	assert.Equal(t, []string{"audit-log-verification", db.AuditVerificationError}, h.sentryEvents[0].Fingerprint)
 	alerts := h.slackAlerts()
 	require.Len(t, alerts, 1)
+	assert.Equal(t, "audit_log.verification_failed", alerts[0].Action)
 	assert.Equal(t, audit.StatusError, alerts[0].Status)
 	assert.Contains(t, alerts[0].Details, "AccessDenied")
 	assert.Equal(t, 1, h.heartbeats, "a run that couldn't verify still ran")

internal/gateway/rbac/audit_names.go

@@ -33,6 +33,10 @@ var auditActions = map[string]string{
 	"POST /api/v1/diagnostics/sentry":                      "integration.sentry_test",
 	"POST /api/v1/settings/rack-tls-cert/refresh":          "rack_tls_cert.refresh",
 	"POST /api/v1/jobs/:id/retry":                          "job.retry",
+	// MFA verification shares auth:mfa_verification:create (audit.ActionMFAVerify and ActionMFAChallenge)
+	"POST /api/v1/auth/mfa/verify":                    "mfa.verify",
+	"POST /api/v1/auth/mfa/webauthn/assertion/verify": "mfa.verify",
+	"POST /api/v1/auth/mfa/webauthn/assertion/start":  "mfa.challenge",
 }
 
 // AuditName is how a route's audit entries name its requests.

internal/gateway/rbac/audit_names_test.go

@@ -62,6 +62,9 @@ func TestAuditNames(t *testing.T) {
 		{"PUT", "/api/v1/settings/mfa-configuration", AuditName{"gateway", "setting.update", "setting"}},
 		{"DELETE", "/api/v1/apps/:app/settings/protected-env-vars", AuditName{"gateway", "setting.delete", "setting"}},
 		{"DELETE", "/api/v1/auth/mfa/methods/:methodID", AuditName{"auth", "mfa_method.delete", "mfa_method"}},
+		{"POST", "/api/v1/auth/mfa/verify", AuditName{"auth", "mfa.verify", "mfa"}},
+		{"POST", "/api/v1/auth/mfa/webauthn/assertion/verify", AuditName{"auth", "mfa.verify", "mfa"}},
+		{"POST", "/api/v1/auth/mfa/webauthn/assertion/start", AuditName{"auth", "mfa.challenge", "mfa"}},
 		{"PUT", "/api/v1/apps/:app/env", AuditName{"convox", "env.set", "env"}},
 		{"GET", "/api/v1/info", AuditName{"gateway", "http.get", "http"}},
 		{"GET", "/api/v1/users", AuditName{"gateway", "user.list", "user"}},

internal/gateway/security/notifier.go

@@ -171,14 +171,15 @@ func (n *Notifier) FailedMFAAttempt(userEmail, userName, ipAddress, userAgent st
 			UserEmail:    userEmail,
 			UserName:     userName,
 			ActionType:   "auth",
-			Action:       "mfa.verify.failed",
+			Action:       audit.ActionMFAVerifyFailed,
 			ResourceType: "auth",
 			Resource:     "mfa",
 			Status:       "failed",
 			IPAddress:    ipAddress,
 			UserAgent:    userAgent,
 		}); err != nil {
-			log.Printf(`{"level":"error","event":"audit_log_failed","action":"mfa.verify.failed","error":%q}`, err)
+			log.Printf(`{"level":"error","event":"audit_log_failed","action":%q,"error":%q}`,
+				audit.ActionMFAVerifyFailed, err)
 		}
 	}
 

internal/gateway/slack/notifier.go

@@ -189,7 +189,7 @@ func isMFAAction(action string) bool {
 	mfaScopes := []string{
 		audit.ActionScopeMFAMethod,
 		audit.ActionScopeMFAPreferences,
-		audit.ActionScopeMFAVerification,
+		audit.ActionScopeMFA,
 		audit.ActionScopeMFABackupCodes,
 		audit.ActionScopeTrustedDevice,
 	}

internal/gateway/slack/notifier_test.go

@@ -88,7 +88,8 @@ func TestMatchGlob(t *testing.T) {
 		want    bool
 	}{
 		{"mfa_method.*", audit.BuildAction(audit.ActionScopeMFAMethod, audit.ActionVerbEnroll), true},
-		{"mfa_method.*", audit.BuildAction(audit.ActionScopeMFAMethod, audit.ActionVerbVerify), true},
+		{"audit_log.verification_*", "audit_log.verification_failed", true},
+		{"audit_log.verification_*", "audit_log.read", false},
 		{"mfa_method.*", "auth.login", false},
 		{"auth.*", "auth.login", true},
 		{"auth.*", "auth.logout", true},