feat: show quota denials in the activity feed

config/milo/activity/policies/connector-policy.yaml

@@ -17,6 +17,13 @@ spec:
     kind: Connector
 
   auditRules:
+    # Create rejected because the project reached its quota. Milo's quota
+    # admission marks these with the quota.miloapis.com/outcome audit annotation.
+    # The resource was never created, so the summary has no link.
+    - name: create-quota-denied
+      match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"
+      summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'connector ' + audit.objectRef.name : 'a connector' }}: quota reached"
+
     # Connector creation with spec available
     - name: create
       match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"

config/milo/activity/policies/connectoradvertisement-policy.yaml

@@ -18,6 +18,13 @@ spec:
     kind: ConnectorAdvertisement
 
   auditRules:
+    # Create rejected because the project reached its quota. Milo's quota
+    # admission marks these with the quota.miloapis.com/outcome audit annotation.
+    # The resource was never created, so the summary has no link.
+    - name: create-quota-denied
+      match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"
+      summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'connector advertisement ' + audit.objectRef.name : 'a connector advertisement' }}: quota reached"
+
     # ConnectorAdvertisement creation with spec available
     - name: create
       match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"

config/milo/activity/policies/gateway-policy.yaml

@@ -17,6 +17,13 @@ spec:
     kind: Gateway
 
   auditRules:
+    # Create rejected because the project reached its quota. Milo's quota
+    # admission marks these with the quota.miloapis.com/outcome audit annotation.
+    # The resource was never created, so the summary has no link.
+    - name: create-quota-denied
+      match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"
+      summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'gateway ' + audit.objectRef.name : 'a gateway' }}: quota reached"
+
     # Gateway creation with spec available
     - name: create
       match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"

config/milo/activity/policies/httpproxy-policy.yaml

@@ -19,6 +19,13 @@ spec:
     kind: HTTPProxy
 
   auditRules:
+    # Create rejected because the project reached its quota. Milo's quota
+    # admission marks these with the quota.miloapis.com/outcome audit annotation.
+    # The resource was never created, so the summary has no link.
+    - name: create-quota-denied
+      match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"
+      summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'load balancer ' + audit.objectRef.name : 'a load balancer' }}: quota reached"
+
     - name: create-annotated-backend
       match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseObject) && type(audit.responseObject) == map && has(audit.responseObject.metadata) && has(audit.responseObject.metadata.annotations) && 'networking.datumapis.com/display-name' in audit.responseObject.metadata.annotations && 'networking.datumapis.com/display-value' in audit.responseObject.metadata.annotations && audit.responseObject.metadata.annotations['networking.datumapis.com/display-value'] != '' && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))"
       summary: "{{ actor }} created load balancer {{ link(audit.responseObject.metadata.annotations['networking.datumapis.com/display-name'], audit.objectRef) }} pointing to {{ audit.responseObject.metadata.annotations['networking.datumapis.com/display-value'] }}"

internal/activitypolicy/quota_denied_policy_test.go

@@ -0,0 +1,73 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package activitypolicy_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// quotaDeniedAudit is the audit event Milo records when quota admission
+// rejects a create: a 403 with a Status body and the outcome annotation.
+func quotaDeniedAudit(name, outcome string) map[string]any {
+	audit := map[string]any{
+		"verb":       "create",
+		"user":       map[string]any{"username": "alice@example.com"},
+		"objectRef":  map[string]any{"name": name, "namespace": "default"},
+		"requestURI": "/apis/networking.datumapis.com/v1alpha/namespaces/default/objects",
+		"requestObject": map[string]any{
+			"metadata": map[string]any{"name": name},
+			"spec":     map[string]any{},
+		},
+		"responseStatus": map[string]any{"code": 403},
+		"responseObject": map[string]any{"kind": "Status", "status": "Failure", "reason": "Forbidden", "code": 403},
+	}
+	if outcome != "" {
+		audit["annotations"] = map[string]any{"quota.miloapis.com/outcome": outcome}
+	}
+	return audit
+}
+
+func TestPolicies_QuotaDenied(t *testing.T) {
+	t.Parallel()
+	policies := []struct {
+		file, named, unnamed string
+	}{
+		{"httpproxy", "Alice couldn't create load balancer obj-1: quota reached", "Alice couldn't create a load balancer: quota reached"},
+		{"connector", "Alice couldn't create connector obj-1: quota reached", "Alice couldn't create a connector: quota reached"},
+		{"gateway", "Alice couldn't create gateway obj-1: quota reached", "Alice couldn't create a gateway: quota reached"},
+		{"connectoradvertisement", "Alice couldn't create connector advertisement obj-1: quota reached", "Alice couldn't create a connector advertisement: quota reached"},
+	}
+	for _, p := range policies {
+		t.Run(p.file, func(t *testing.T) {
+			t.Parallel()
+			pol := loadPolicy(t, "config/milo/activity/policies/"+p.file+"-policy.yaml")
+
+			audit := quotaDeniedAudit("obj-1", "denied")
+			require.Equal(t, "create-quota-denied", firstMatchingAuditRule(t, pol, audit))
+			assert.Equal(t, p.named, auditSummary(t, pol, "create-quota-denied", audit))
+
+			// generateName creates have no name yet.
+			unnamed := quotaDeniedAudit("", "denied")
+			require.Equal(t, "create-quota-denied", firstMatchingAuditRule(t, pol, unnamed))
+			assert.Equal(t, p.unnamed, auditSummary(t, pol, "create-quota-denied", unnamed))
+
+			for name, a := range map[string]map[string]any{
+				// A permissions failure is also a 403, without the annotation.
+				"forbidden without quota annotation": quotaDeniedAudit("obj-1", ""),
+				// Timeouts and other failures are platform problems, not the
+				// customer reaching a limit.
+				"quota check timed out": quotaDeniedAudit("obj-1", "timeout"),
+				"quota internal error":  quotaDeniedAudit("obj-1", "internal_error"),
+			} {
+				assert.Empty(t, firstMatchingAuditRule(t, pol, a), name)
+			}
+
+			system := quotaDeniedAudit("obj-1", "denied")
+			system["user"] = map[string]any{"username": "system:control@networking.datumapis.com"}
+			assert.Empty(t, firstMatchingAuditRule(t, pol, system), "controller-side denials stay out of the feed")
+		})
+	}
+}

test/activitypolicy/policies_test.go

@@ -69,6 +69,12 @@ func gatesOn2xx(match string) bool {
 		strings.Contains(match, "audit.responseStatus.code < 300")
 }
 
+// matchesQuotaDenial reports whether a rule records a create that Milo's
+// quota admission rejected. These rules match a 403 on purpose.
+func matchesQuotaDenial(match string) bool {
+	return strings.Contains(match, "audit.annotations['quota.miloapis.com/outcome'] == 'denied'")
+}
+
 func skipsDryRun(match string) bool {
 	return strings.Contains(match, "audit.requestURI.contains('dryRun=')")
 }
@@ -186,7 +192,7 @@ func TestWriteRulesGateOnOutcome(t *testing.T) {
 	for _, pol := range loadPolicies(t) {
 		for _, r := range pol.Spec.AuditRules {
 			v := verbOf(r.Match)
-			if v == "other" {
+			if v == "other" || matchesQuotaDenial(r.Match) {
 				continue
 			}
 			t.Run(pol.Name+"/"+r.Name, func(t *testing.T) {
@@ -211,7 +217,7 @@ func TestWriteRulesFireOnlyOnSuccess(t *testing.T) {
 	for _, pol := range loadPolicies(t) {
 		for _, r := range pol.Spec.AuditRules {
 			v := verbOf(r.Match)
-			if v == "other" {
+			if v == "other" || matchesQuotaDenial(r.Match) {
 				continue
 			}
 			t.Run(pol.Name+"/"+r.Name, func(t *testing.T) {