From 6f7c7c0bdfc9c1b0fd64e01e21db23d085928905 Mon Sep 17 00:00:00 2001 From: Kevin Williams Date: Wed, 30 Sep 2026 19:54:22 -0700 Subject: [PATCH] feat: show quota denials in the activity feed Adds a create-quota-denied rule to the HTTPProxy, Connector, Gateway and ConnectorAdvertisement policies. It matches creates that Milo's quota admission rejected, using the quota.miloapis.com/outcome audit annotation (milo-os/milo#821), and skips system users, other 403s and quota timeouts. --- .../activity/policies/connector-policy.yaml | 7 ++ .../connectoradvertisement-policy.yaml | 7 ++ .../activity/policies/gateway-policy.yaml | 7 ++ .../activity/policies/httpproxy-policy.yaml | 7 ++ .../quota_denied_policy_test.go | 73 +++++++++++++++++++ test/activitypolicy/policies_test.go | 10 ++- 6 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 internal/activitypolicy/quota_denied_policy_test.go diff --git a/config/milo/activity/policies/connector-policy.yaml b/config/milo/activity/policies/connector-policy.yaml index 702c8503..9e5f1055 100644 --- a/config/milo/activity/policies/connector-policy.yaml +++ b/config/milo/activity/policies/connector-policy.yaml @@ -17,6 +17,13 @@ spec: kind: Connector auditRules: + # Create rejected because the project reached its quota. Milo's quota + # admission marks these with the quota.miloapis.com/outcome audit annotation. + # The resource was never created, so the summary has no link. + - name: create-quota-denied + match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" + summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'connector ' + audit.objectRef.name : 'a connector' }}: quota reached" + # Connector creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" diff --git a/config/milo/activity/policies/connectoradvertisement-policy.yaml b/config/milo/activity/policies/connectoradvertisement-policy.yaml index 2e86a260..3e400ffa 100644 --- a/config/milo/activity/policies/connectoradvertisement-policy.yaml +++ b/config/milo/activity/policies/connectoradvertisement-policy.yaml @@ -18,6 +18,13 @@ spec: kind: ConnectorAdvertisement auditRules: + # Create rejected because the project reached its quota. Milo's quota + # admission marks these with the quota.miloapis.com/outcome audit annotation. + # The resource was never created, so the summary has no link. + - name: create-quota-denied + match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" + summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'connector advertisement ' + audit.objectRef.name : 'a connector advertisement' }}: quota reached" + # ConnectorAdvertisement creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" diff --git a/config/milo/activity/policies/gateway-policy.yaml b/config/milo/activity/policies/gateway-policy.yaml index 779ff47a..534cac7d 100644 --- a/config/milo/activity/policies/gateway-policy.yaml +++ b/config/milo/activity/policies/gateway-policy.yaml @@ -17,6 +17,13 @@ spec: kind: Gateway auditRules: + # Create rejected because the project reached its quota. Milo's quota + # admission marks these with the quota.miloapis.com/outcome audit annotation. + # The resource was never created, so the summary has no link. + - name: create-quota-denied + match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" + summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'gateway ' + audit.objectRef.name : 'a gateway' }}: quota reached" + # Gateway creation with spec available - name: create match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" diff --git a/config/milo/activity/policies/httpproxy-policy.yaml b/config/milo/activity/policies/httpproxy-policy.yaml index ef54f9fd..4d07a8fa 100644 --- a/config/milo/activity/policies/httpproxy-policy.yaml +++ b/config/milo/activity/policies/httpproxy-policy.yaml @@ -19,6 +19,13 @@ spec: kind: HTTPProxy auditRules: + # Create rejected because the project reached its quota. Milo's quota + # admission marks these with the quota.miloapis.com/outcome audit annotation. + # The resource was never created, so the summary has no link. + - name: create-quota-denied + match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.responseStatus.code) && audit.responseStatus.code == 403 && has(audit.annotations) && 'quota.miloapis.com/outcome' in audit.annotations && audit.annotations['quota.miloapis.com/outcome'] == 'denied' && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" + summary: "{{ actor }} couldn't create {{ has(audit.objectRef.name) && audit.objectRef.name != '' ? 'load balancer ' + audit.objectRef.name : 'a load balancer' }}: quota reached" + - name: create-annotated-backend match: "!audit.user.username.startsWith('system:') && audit.verb == 'create' && has(audit.requestObject.spec) && has(audit.responseObject) && type(audit.responseObject) == map && has(audit.responseObject.metadata) && has(audit.responseObject.metadata.annotations) && 'networking.datumapis.com/display-name' in audit.responseObject.metadata.annotations && 'networking.datumapis.com/display-value' in audit.responseObject.metadata.annotations && audit.responseObject.metadata.annotations['networking.datumapis.com/display-value'] != '' && has(audit.responseStatus.code) && audit.responseStatus.code >= 200 && audit.responseStatus.code < 300 && !(has(audit.requestURI) && audit.requestURI.contains('dryRun='))" summary: "{{ actor }} created load balancer {{ link(audit.responseObject.metadata.annotations['networking.datumapis.com/display-name'], audit.objectRef) }} pointing to {{ audit.responseObject.metadata.annotations['networking.datumapis.com/display-value'] }}" diff --git a/internal/activitypolicy/quota_denied_policy_test.go b/internal/activitypolicy/quota_denied_policy_test.go new file mode 100644 index 00000000..7a351e3a --- /dev/null +++ b/internal/activitypolicy/quota_denied_policy_test.go @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package activitypolicy_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// quotaDeniedAudit is the audit event Milo records when quota admission +// rejects a create: a 403 with a Status body and the outcome annotation. +func quotaDeniedAudit(name, outcome string) map[string]any { + audit := map[string]any{ + "verb": "create", + "user": map[string]any{"username": "alice@example.com"}, + "objectRef": map[string]any{"name": name, "namespace": "default"}, + "requestURI": "/apis/networking.datumapis.com/v1alpha/namespaces/default/objects", + "requestObject": map[string]any{ + "metadata": map[string]any{"name": name}, + "spec": map[string]any{}, + }, + "responseStatus": map[string]any{"code": 403}, + "responseObject": map[string]any{"kind": "Status", "status": "Failure", "reason": "Forbidden", "code": 403}, + } + if outcome != "" { + audit["annotations"] = map[string]any{"quota.miloapis.com/outcome": outcome} + } + return audit +} + +func TestPolicies_QuotaDenied(t *testing.T) { + t.Parallel() + policies := []struct { + file, named, unnamed string + }{ + {"httpproxy", "Alice couldn't create load balancer obj-1: quota reached", "Alice couldn't create a load balancer: quota reached"}, + {"connector", "Alice couldn't create connector obj-1: quota reached", "Alice couldn't create a connector: quota reached"}, + {"gateway", "Alice couldn't create gateway obj-1: quota reached", "Alice couldn't create a gateway: quota reached"}, + {"connectoradvertisement", "Alice couldn't create connector advertisement obj-1: quota reached", "Alice couldn't create a connector advertisement: quota reached"}, + } + for _, p := range policies { + t.Run(p.file, func(t *testing.T) { + t.Parallel() + pol := loadPolicy(t, "config/milo/activity/policies/"+p.file+"-policy.yaml") + + audit := quotaDeniedAudit("obj-1", "denied") + require.Equal(t, "create-quota-denied", firstMatchingAuditRule(t, pol, audit)) + assert.Equal(t, p.named, auditSummary(t, pol, "create-quota-denied", audit)) + + // generateName creates have no name yet. + unnamed := quotaDeniedAudit("", "denied") + require.Equal(t, "create-quota-denied", firstMatchingAuditRule(t, pol, unnamed)) + assert.Equal(t, p.unnamed, auditSummary(t, pol, "create-quota-denied", unnamed)) + + for name, a := range map[string]map[string]any{ + // A permissions failure is also a 403, without the annotation. + "forbidden without quota annotation": quotaDeniedAudit("obj-1", ""), + // Timeouts and other failures are platform problems, not the + // customer reaching a limit. + "quota check timed out": quotaDeniedAudit("obj-1", "timeout"), + "quota internal error": quotaDeniedAudit("obj-1", "internal_error"), + } { + assert.Empty(t, firstMatchingAuditRule(t, pol, a), name) + } + + system := quotaDeniedAudit("obj-1", "denied") + system["user"] = map[string]any{"username": "system:control@networking.datumapis.com"} + assert.Empty(t, firstMatchingAuditRule(t, pol, system), "controller-side denials stay out of the feed") + }) + } +} diff --git a/test/activitypolicy/policies_test.go b/test/activitypolicy/policies_test.go index 2cbeaf3f..a0058113 100644 --- a/test/activitypolicy/policies_test.go +++ b/test/activitypolicy/policies_test.go @@ -69,6 +69,12 @@ func gatesOn2xx(match string) bool { strings.Contains(match, "audit.responseStatus.code < 300") } +// matchesQuotaDenial reports whether a rule records a create that Milo's +// quota admission rejected. These rules match a 403 on purpose. +func matchesQuotaDenial(match string) bool { + return strings.Contains(match, "audit.annotations['quota.miloapis.com/outcome'] == 'denied'") +} + func skipsDryRun(match string) bool { return strings.Contains(match, "audit.requestURI.contains('dryRun=')") } @@ -186,7 +192,7 @@ func TestWriteRulesGateOnOutcome(t *testing.T) { for _, pol := range loadPolicies(t) { for _, r := range pol.Spec.AuditRules { v := verbOf(r.Match) - if v == "other" { + if v == "other" || matchesQuotaDenial(r.Match) { continue } t.Run(pol.Name+"/"+r.Name, func(t *testing.T) { @@ -211,7 +217,7 @@ func TestWriteRulesFireOnlyOnSuccess(t *testing.T) { for _, pol := range loadPolicies(t) { for _, r := range pol.Spec.AuditRules { v := verbOf(r.Match) - if v == "other" { + if v == "other" || matchesQuotaDenial(r.Match) { continue } t.Run(pol.Name+"/"+r.Name, func(t *testing.T) {