From 050448fcf4f589c09ab81efa77dbe2bd1cb9157f Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:24:45 +0000 Subject: [PATCH 01/10] Add repo-based cab allowlist datastore sync tool --- tools/repo-cab-sync/README.md | 56 ++++++ tools/repo-cab-sync/go.mod | 41 +++++ tools/repo-cab-sync/go.sum | 101 ++++++++++ tools/repo-cab-sync/main.go | 173 ++++++++++++++++++ tools/repo-cab-sync/main_test.go | 105 +++++++++++ tools/repo-cab-sync/repo_cab_allowlist.yaml | 7 + .../repo_cab_allowlist_test.yaml | 7 + 7 files changed, 490 insertions(+) create mode 100644 tools/repo-cab-sync/README.md create mode 100644 tools/repo-cab-sync/go.mod create mode 100644 tools/repo-cab-sync/go.sum create mode 100644 tools/repo-cab-sync/main.go create mode 100644 tools/repo-cab-sync/main_test.go create mode 100644 tools/repo-cab-sync/repo_cab_allowlist.yaml create mode 100644 tools/repo-cab-sync/repo_cab_allowlist_test.yaml diff --git a/tools/repo-cab-sync/README.md b/tools/repo-cab-sync/README.md new file mode 100644 index 00000000000..de24c7f6431 --- /dev/null +++ b/tools/repo-cab-sync/README.md @@ -0,0 +1,56 @@ +# Repo Consider All Branches Allowlist Sync Tool (`repo-cab-sync`) + +`repo-cab-sync` is a Go command-line tool that synchronizes repository "Consider All Branches" (CAB) allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoConsiderAllBranchesAllowList` entities). + +## Overview + +Gitter has the option to enumerate affected commits with `consider_all_branches` enabled or disabled. This tool manages the Datastore allowlist index (`RepoConsiderAllBranchesAllowList`) that controls this behavior on a repository level. + +> [!NOTE] +> If `consider_all_branches` is already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. + +The tool performs a two-way sync: + +- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. +- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. + +## Allowlist YAML Format + +The allowlist YAML configuration file accepts a list of entries with `type` and `value` fields: + +```yaml +# Supported entry types: 'url' and 'regex' + +# Exact repository URL match +- type: url + value: "https://github.com/google/osv.dev.git" + +# Regex pattern match (Go RE2 syntax) +- type: regex + value: 'github\.com/google/osv-.*' +``` + +> [!TIP] +> Use single quotes for regex values so you don't have to escape backslashes or other special characters. + +### Normalization and Validation + +- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. + +## Usage + +Run the tool using `go run`: + +```bash +go run . [flags] +``` + +### Options & Flags + +| Flag | Default | Description | +| ----------- | ------------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_cab_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-cab-sync/go.mod new file mode 100644 index 00000000000..4595f7b868a --- /dev/null +++ b/tools/repo-cab-sync/go.mod @@ -0,0 +1,41 @@ +module github.com/google/osv.dev/tools/repo-cab-sync + +go 1.26.5 + +require ( + cloud.google.com/go/datastore v1.25.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.23.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.287.1 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/grpc v1.82.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect +) diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-cab-sync/go.sum new file mode 100644 index 00000000000..360fa03b5ce --- /dev/null +++ b/tools/repo-cab-sync/go.sum @@ -0,0 +1,101 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= +cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= +github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= +google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= +google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go new file mode 100644 index 00000000000..44dcba17a8c --- /dev/null +++ b/tools/repo-cab-sync/main.go @@ -0,0 +1,173 @@ +// Package main implements a CLI tool to sync Repo Consider All Branches (CAB) allowlist YAML files to Cloud Datastore. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + + "cloud.google.com/go/datastore" + "gopkg.in/yaml.v3" +) + +type RepoCABEntity struct { + Key *datastore.Key `yaml:"-" datastore:"__key__"` + Type string `yaml:"type" datastore:"type"` + Value string `yaml:"value" datastore:"value"` +} + +func main() { + filePath := flag.String("file", "repo_cab_allowlist.yaml", "Path to repo_cab_allowlist YAML file") + project := flag.String("project", "oss-vdb-test", "GCP project ID") + dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") + verbose := flag.Bool("verbose", false, "Display verbose sync operations") + + flag.Parse() + + if *filePath == "" { + log.Fatalf("Error: --file argument is required") + } + + if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + log.Fatalf("Error syncing repo CAB allowlist: %v", err) + } +} + +func normalizeRepo(repoURL string) string { + // Normalize the repo_url to align with matching logic + // Removes the scheme/protocol, the .git extension, and trailing slashes. + if repoURL == "" { + return "" + } + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { + var parsed []RepoCABEntity + if err := yaml.Unmarshal(data, &parsed); err != nil { + return nil, err + } + + var entries []RepoCABEntity + for _, entry := range parsed { + if entry.Type == "url" { + // For repo URLs, we normalize the value before inserting to datastore + entry.Value = normalizeRepo(entry.Value) + } else if entry.Type == "regex" { + // For regex, we make sure it compiles + if _, err := regexp.Compile(entry.Value); err != nil { + log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) + continue + } + } + entries = append(entries, entry) + } + + return entries, nil +} + +func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed reading file %s: %w", filePath, err) + } + + entries, err := parseYAMLEntries(data) + if err != nil { + return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) + } + + if verbose { + log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) + } + + dsClient, err := datastore.NewClient(ctx, project) + if err != nil { + return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) + } + defer func() { _ = dsClient.Close() }() + + // Get existing Datastore entities + query := datastore.NewQuery("RepoConsiderAllBranchesAllowList") + var dsEntities []RepoCABEntity + if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) + } + + dsEntitiesMap := make(map[string]RepoCABEntity) + for _, entity := range dsEntities { + dsEntitiesMap[entity.Value] = entity + } + + localEntriesMap := make(map[string]RepoCABEntity) + for _, item := range entries { + localEntriesMap[item.Value] = item + } + + // 1. Put/Upsert entries in local YAML that are not in Datastore or modified + for val, item := range localEntriesMap { + existing, exists := dsEntitiesMap[val] + if !exists { + key := datastore.IncompleteKey("RepoConsiderAllBranchesAllowList", nil) + entity := &RepoCABEntity{ + Type: item.Type, + Value: item.Value, + } + if !dryRun { + if _, err := dsClient.Put(ctx, key, entity); err != nil { + return fmt.Errorf("failed putting entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Creating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } else if existing.Type != item.Type { + entity := &RepoCABEntity{ + Type: item.Type, + Value: item.Value, + } + if !dryRun { + if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + return fmt.Errorf("failed updating entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Updating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } + } + + // 2. Delete entries in Datastore that are no longer in local YAML + for val, existing := range dsEntitiesMap { + if _, exists := localEntriesMap[val]; !exists { + if verbose { + log.Printf("Deleting RepoConsiderAllBranchesAllowList entity: val=%s", val) + } + if !dryRun { + if err := dsClient.Delete(ctx, existing.Key); err != nil { + return fmt.Errorf("failed deleting entity for %s: %w", val, err) + } + } + } + } + + if dryRun { + log.Println("[DRY RUN] Sync completed successfully.") + } else { + log.Println("Sync completed successfully.") + } + return nil +} diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-cab-sync/main_test.go new file mode 100644 index 00000000000..b06be9414ca --- /dev/null +++ b/tools/repo-cab-sync/main_test.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestNormalizeRepo(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "Empty string", + input: "", + expected: "", + }, + { + name: "HTTPS URL with .git", + input: "https://github.com/google/osv.dev.git", + expected: "github.com/google/osv.dev", + }, + { + name: "HTTPS URL without .git", + input: "https://github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "URL with trailing slash", + input: "https://github.com/google/osv.dev/", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme URL", + input: "github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + }, + { + name: "No scheme with .git", + input: "github.com/google/osv-scanner.git", + expected: "github.com/google/osv-scanner", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := normalizeRepo(tt.input) + if got != tt.expected { + t.Errorf("normalizeRepo(%q) = %q, want %q", tt.input, got, tt.expected) + } + }) + } +} + +func TestParseYAMLEntries(t *testing.T) { + yamlContent := []byte(` +- type: url + value: "https://github.com/google/osv.dev.git" +- type: regex + value: 'github\.com/google/osv-.*' +- type: regex + value: '[invalid regex' +`) + + entries, err := parseYAMLEntries(yamlContent) + if err != nil { + t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) + } + + // Should skip invalid regex and return 2 entries + if len(entries) != 2 { + t.Fatalf("expected 2 valid entries, got %d", len(entries)) + } + + // First entry should be normalized URL + if entries[0].Type != "url" || entries[0].Value != "github.com/google/osv.dev" { + t.Errorf("entry 0 mismatch: got type=%q val=%q, want type=url val=github.com/google/osv.dev", entries[0].Type, entries[0].Value) + } + + // Second entry should preserve regex string + if entries[1].Type != "regex" || entries[1].Value != `github\.com/google/osv-.*` { + t.Errorf("entry 1 mismatch: got type=%q val=%q, want type=regex val=github\\.com/google/osv-.*", entries[1].Type, entries[1].Value) + } +} + +func TestRun_InvalidFile(t *testing.T) { + err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false) + if err == nil { + t.Error("expected error for non-existent file, got nil") + } + + tmpDir := t.TempDir() + badYAMLPath := filepath.Join(tmpDir, "bad.yaml") + if err := os.WriteFile(badYAMLPath, []byte("invalid: yaml: ["), 0644); err != nil { + t.Fatalf("failed creating bad yaml file: %v", err) + } + + err = run(context.Background(), badYAMLPath, "test-project", true, false) + if err == nil { + t.Error("expected error for invalid YAML file, got nil") + } +} diff --git a/tools/repo-cab-sync/repo_cab_allowlist.yaml b/tools/repo-cab-sync/repo_cab_allowlist.yaml new file mode 100644 index 00000000000..2b477302dc5 --- /dev/null +++ b/tools/repo-cab-sync/repo_cab_allowlist.yaml @@ -0,0 +1,7 @@ +# Repository-based consider all branches allowlist +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# - type: regex +# value: 'github\.com/google/osv-.*' diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml new file mode 100644 index 00000000000..2b477302dc5 --- /dev/null +++ b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml @@ -0,0 +1,7 @@ +# Repository-based consider all branches allowlist +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# - type: regex +# value: 'github\.com/google/osv-.*' From fd946a0e0614e4d77df4bff719bccbcafb1b552b Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:50:22 +0000 Subject: [PATCH 02/10] update some dependencies --- tools/repo-cab-sync/go.mod | 10 +++++----- tools/repo-cab-sync/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-cab-sync/go.mod index 4595f7b868a..c316e0eba69 100644 --- a/tools/repo-cab-sync/go.mod +++ b/tools/repo-cab-sync/go.mod @@ -25,17 +25,17 @@ require ( go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - golang.org/x/crypto v0.53.0 // indirect + golang.org/x/crypto v0.54.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.21.0 // indirect - golang.org/x/sys v0.46.0 // indirect - golang.org/x/text v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.287.1 // indirect google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/grpc v1.82.0 // indirect + google.golang.org/grpc v1.82.1 // indirect google.golang.org/protobuf v1.36.11 // indirect ) diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-cab-sync/go.sum index 360fa03b5ce..7c948c6563f 100644 --- a/tools/repo-cab-sync/go.sum +++ b/tools/repo-cab-sync/go.sum @@ -66,18 +66,18 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= @@ -90,8 +90,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= -google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 1b321ffcca2b18819832b7fab00dd2f7c963ef65 Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 01:53:00 +0000 Subject: [PATCH 03/10] populate a repo in test --- tools/repo-cab-sync/repo_cab_allowlist_test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml index 2b477302dc5..4e83defe770 100644 --- a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml +++ b/tools/repo-cab-sync/repo_cab_allowlist_test.yaml @@ -5,3 +5,5 @@ # value: "https://github.com/google/osv.dev.git" # - type: regex # value: 'github\.com/google/osv-.*' +- type: url + value: 'https://github.com/apache/hadoop.git' From 03dd4983cb966709045ed15aa04125acfeb98456 Mon Sep 17 00:00:00 2001 From: Joey L Date: Thu, 6 Aug 2026 02:03:49 +0000 Subject: [PATCH 04/10] warn if not url or regex type --- tools/repo-cab-sync/main.go | 24 ++++++++++++++++++++++-- tools/repo-cab-sync/main_test.go | 29 ++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go index 44dcba17a8c..7ab771a54c5 100644 --- a/tools/repo-cab-sync/main.go +++ b/tools/repo-cab-sync/main.go @@ -41,9 +41,16 @@ func main() { func normalizeRepo(repoURL string) string { // Normalize the repo_url to align with matching logic // Removes the scheme/protocol, the .git extension, and trailing slashes. + repoURL = strings.TrimSpace(repoURL) if repoURL == "" { return "" } + + if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { + log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) + return "" + } + parsed, err := url.Parse(repoURL) if err != nil { return repoURL @@ -63,16 +70,29 @@ func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { var entries []RepoCABEntity for _, entry := range parsed { - if entry.Type == "url" { + entry.Type = strings.TrimSpace(strings.ToLower(entry.Type)) + + switch entry.Type { + case "url": // For repo URLs, we normalize the value before inserting to datastore entry.Value = normalizeRepo(entry.Value) - } else if entry.Type == "regex" { + if entry.Value == "" { + continue + } + case "regex": // For regex, we make sure it compiles + if entry.Value == "" { + continue + } if _, err := regexp.Compile(entry.Value); err != nil { log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) continue } + default: + log.Printf("Warning: Skipping unrecognized entry type %q for value %q", entry.Type, entry.Value) + continue } + entries = append(entries, entry) } diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-cab-sync/main_test.go index b06be9414ca..c7c5a8cca45 100644 --- a/tools/repo-cab-sync/main_test.go +++ b/tools/repo-cab-sync/main_test.go @@ -43,6 +43,21 @@ func TestNormalizeRepo(t *testing.T) { input: "github.com/google/osv-scanner.git", expected: "github.com/google/osv-scanner", }, + { + name: "Whitespace in URL", + input: " https://github.com/google/osv.dev.git ", + expected: "github.com/google/osv.dev", + }, + { + name: "SSH URL format git@", + input: "git@github.com:google/osv.dev.git", + expected: "", + }, + { + name: "SSH URL format ssh://", + input: "ssh://git@github.com/google/osv.dev.git", + expected: "", + }, } for _, tt := range tests { @@ -57,12 +72,16 @@ func TestNormalizeRepo(t *testing.T) { func TestParseYAMLEntries(t *testing.T) { yamlContent := []byte(` -- type: url - value: "https://github.com/google/osv.dev.git" -- type: regex +- type: URL + value: " https://github.com/google/osv.dev.git " +- type: REGEX value: 'github\.com/google/osv-.*' - type: regex value: '[invalid regex' +- type: unknown + value: "https://github.com/google/osv.dev" +- type: url + value: "git@github.com:google/osv.dev.git" `) entries, err := parseYAMLEntries(yamlContent) @@ -70,9 +89,9 @@ func TestParseYAMLEntries(t *testing.T) { t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) } - // Should skip invalid regex and return 2 entries + // Should normalize types, trim values, and skip invalid regex, unrecognized types, and SSH URLs if len(entries) != 2 { - t.Fatalf("expected 2 valid entries, got %d", len(entries)) + t.Fatalf("expected 2 valid entries, got %d: %+v", len(entries), entries) } // First entry should be normalized URL From 8352fb8d030e1c4892324679b3572350c3f93c82 Mon Sep 17 00:00:00 2001 From: Joey L Date: Fri, 7 Aug 2026 05:13:39 +0000 Subject: [PATCH 05/10] Expand cab allowlist to cherrypick options --- tools/repo-allowlist-sync/README.md | 70 +++++ .../go.mod | 2 +- .../go.sum | 0 tools/repo-allowlist-sync/main.go | 245 ++++++++++++++++++ .../main_test.go | 56 +++- tools/repo-allowlist-sync/repo_allowlist.yaml | 15 ++ .../repo_allowlist_test.yaml} | 6 +- tools/repo-cab-sync/README.md | 56 ---- tools/repo-cab-sync/main.go | 193 -------------- tools/repo-cab-sync/repo_cab_allowlist.yaml | 7 - 10 files changed, 379 insertions(+), 271 deletions(-) create mode 100644 tools/repo-allowlist-sync/README.md rename tools/{repo-cab-sync => repo-allowlist-sync}/go.mod (96%) rename tools/{repo-cab-sync => repo-allowlist-sync}/go.sum (100%) create mode 100644 tools/repo-allowlist-sync/main.go rename tools/{repo-cab-sync => repo-allowlist-sync}/main_test.go (65%) create mode 100644 tools/repo-allowlist-sync/repo_allowlist.yaml rename tools/{repo-cab-sync/repo_cab_allowlist_test.yaml => repo-allowlist-sync/repo_allowlist_test.yaml} (65%) delete mode 100644 tools/repo-cab-sync/README.md delete mode 100644 tools/repo-cab-sync/main.go delete mode 100644 tools/repo-cab-sync/repo_cab_allowlist.yaml diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md new file mode 100644 index 00000000000..ebed96ee8a7 --- /dev/null +++ b/tools/repo-allowlist-sync/README.md @@ -0,0 +1,70 @@ +# Repository Allowlist Sync Tool (`repo-allowlist-sync`) + +`repo-allowlist-sync` is a Go command-line tool that synchronizes repository allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoAllowList` entities). + +## Overview + +Gitter has options to enumerate affected commits with `consider_all_branches` and cherrypick detection options (`cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`). This tool manages the Datastore allowlist index (`RepoAllowList`) that controls these behaviors on a repository level. + +> [!NOTE] +> If feature flags are already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. + +The tool performs a two-way sync: + +- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. +- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. + +## Allowlist YAML Format + +The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields: + +```yaml +# Supported entry types: 'url' and 'regex' + +# Shorthand: 'cherrypicks: true' applies to all 3 cherrypick flags (introduced, fixed, limit) +- type: url + value: "https://github.com/google/osv.dev.git" + consider_all_branches: true + cherrypicks: true + +# Fine-grained control with specific overrides +- type: url + value: "https://github.com/apache/hadoop.git" + consider_all_branches: true + cherrypicks: true + cherrypicks_fixed: false # Specific override for fixed + +# Regex pattern match (Go RE2 syntax) +- type: regex + value: 'github\.com/google/osv-.*' + consider_all_branches: true + cherrypicks_introduced: true + cherrypicks_fixed: true + cherrypicks_limit: true +``` + +> [!TIP] +> Use single quotes for regex values so you don't have to escape backslashes or other special characters. + +### Normalization and Validation + +- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. +- **`cherrypicks`**: Acts as a shorthand for setting `cherrypicks_introduced`, `cherrypicks_fixed`, and `cherrypicks_limit` simultaneously. Specific `cherrypicks_` fields override the shorthand value if provided. + +## Usage + +Run the tool using `go run`: + +```bash +go run . [flags] +``` + +### Options & Flags + +| Flag | Default | Description | +| ----------- | --------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/go.mod b/tools/repo-allowlist-sync/go.mod similarity index 96% rename from tools/repo-cab-sync/go.mod rename to tools/repo-allowlist-sync/go.mod index c316e0eba69..941de59e832 100644 --- a/tools/repo-cab-sync/go.mod +++ b/tools/repo-allowlist-sync/go.mod @@ -1,4 +1,4 @@ -module github.com/google/osv.dev/tools/repo-cab-sync +module github.com/google/osv.dev/tools/repo-allowlist-sync go 1.26.5 diff --git a/tools/repo-cab-sync/go.sum b/tools/repo-allowlist-sync/go.sum similarity index 100% rename from tools/repo-cab-sync/go.sum rename to tools/repo-allowlist-sync/go.sum diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go new file mode 100644 index 00000000000..d8a0a0bf2bb --- /dev/null +++ b/tools/repo-allowlist-sync/main.go @@ -0,0 +1,245 @@ +// Package main implements a CLI tool to sync Repository AllowList YAML files to Cloud Datastore. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "os" + "regexp" + "strings" + + "cloud.google.com/go/datastore" + "gopkg.in/yaml.v3" +) + +// RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore +type RepoAllowListEntity struct { + Key *datastore.Key `datastore:"__key__"` + Type string `datastore:"type"` + Value string `datastore:"value"` + ConsiderAllBranches bool `datastore:"consider_all_branches"` + CherrypicksIntroduced bool `datastore:"cherrypicks_introduced"` + CherrypicksFixed bool `datastore:"cherrypicks_fixed"` + CherrypicksLimit bool `datastore:"cherrypicks_limit"` +} + +// rawYAMLEntry represents an unmarshaled entry from the YAML file, including optional shorthand field +type rawYAMLEntry struct { + Type string `yaml:"type"` + Value string `yaml:"value"` + ConsiderAllBranches bool `yaml:"consider_all_branches"` + Cherrypicks *bool `yaml:"cherrypicks"` + CherrypicksIntroduced *bool `yaml:"cherrypicks_introduced"` + CherrypicksFixed *bool `yaml:"cherrypicks_fixed"` + CherrypicksLimit *bool `yaml:"cherrypicks_limit"` +} + +func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { + return e.Type == other.Type && + e.ConsiderAllBranches == other.ConsiderAllBranches && + e.CherrypicksIntroduced == other.CherrypicksIntroduced && + e.CherrypicksFixed == other.CherrypicksFixed && + e.CherrypicksLimit == other.CherrypicksLimit +} + +func main() { + filePath := flag.String("file", "repo_allowlist.yaml", "Path to repo_allowlist YAML file") + project := flag.String("project", "oss-vdb-test", "GCP project ID") + dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") + verbose := flag.Bool("verbose", false, "Display verbose sync operations") + + flag.Parse() + + if *filePath == "" { + log.Fatalf("Error: --file argument is required") + } + + if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + log.Fatalf("Error syncing repo allowlist: %v", err) + } +} + +// normalizeRepo removes the URL scheme, trailing slashes, and .git extensions to standardize repo paths. +func normalizeRepo(repoURL string) string { + repoURL = strings.TrimSpace(repoURL) + if repoURL == "" { + return "" + } + + if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { + log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) + return "" + } + + parsed, err := url.Parse(repoURL) + if err != nil { + return repoURL + } + normalized := parsed.Host + parsed.Path + normalized = strings.TrimRight(normalized, "/") + normalized = strings.TrimSuffix(normalized, ".git") + + return normalized +} + +// parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. +func parseYAMLEntries(data []byte) ([]RepoAllowListEntity, error) { + var rawEntries []rawYAMLEntry + if err := yaml.Unmarshal(data, &rawEntries); err != nil { + return nil, err + } + + var entries []RepoAllowListEntity + for _, raw := range rawEntries { + raw.Type = strings.TrimSpace(strings.ToLower(raw.Type)) + + switch raw.Type { + case "url": + raw.Value = normalizeRepo(raw.Value) + if raw.Value == "" { + continue + } + case "regex": + if raw.Value == "" { + continue + } + if _, err := regexp.Compile(raw.Value); err != nil { + log.Printf("Warning: Skipping invalid regex pattern %q: %v", raw.Value, err) + continue + } + default: + log.Printf("Warning: Skipping unrecognized entry type %q for value %q", raw.Type, raw.Value) + continue + } + + // Process cherrypicks flags: "cherrypicks: bool" acts as a shorthand for all 3 event types, + // specifying "cherrypicks_" fields overrides that. + intro := false + fixed := false + limit := false + + if raw.Cherrypicks != nil { + intro = *raw.Cherrypicks + fixed = *raw.Cherrypicks + limit = *raw.Cherrypicks + } + if raw.CherrypicksIntroduced != nil { + intro = *raw.CherrypicksIntroduced + } + if raw.CherrypicksFixed != nil { + fixed = *raw.CherrypicksFixed + } + if raw.CherrypicksLimit != nil { + limit = *raw.CherrypicksLimit + } + + entries = append(entries, RepoAllowListEntity{ + Type: raw.Type, + Value: raw.Value, + ConsiderAllBranches: raw.ConsiderAllBranches, + CherrypicksIntroduced: intro, + CherrypicksFixed: fixed, + CherrypicksLimit: limit, + }) + } + + return entries, nil +} + +func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { + data, err := os.ReadFile(filePath) + if err != nil { + return fmt.Errorf("failed reading file %s: %w", filePath, err) + } + + entries, err := parseYAMLEntries(data) + if err != nil { + return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) + } + + if verbose { + log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) + } + + dsClient, err := datastore.NewClient(ctx, project) + if err != nil { + return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) + } + defer func() { _ = dsClient.Close() }() + + // Fetch existing Datastore entities + query := datastore.NewQuery("RepoAllowList") + var dsEntities []RepoAllowListEntity + if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) + } + + dsEntitiesMap := make(map[string]RepoAllowListEntity) + for _, entity := range dsEntities { + dsEntitiesMap[entity.Value] = entity + } + + localEntriesMap := make(map[string]RepoAllowListEntity) + for _, item := range entries { + localEntriesMap[item.Value] = item + } + + // Upsert entries in local YAML that are missing from Datastore or modified + for val, item := range localEntriesMap { + existing, exists := dsEntitiesMap[val] + entity := &RepoAllowListEntity{ + Type: item.Type, + Value: item.Value, + ConsiderAllBranches: item.ConsiderAllBranches, + CherrypicksIntroduced: item.CherrypicksIntroduced, + CherrypicksFixed: item.CherrypicksFixed, + CherrypicksLimit: item.CherrypicksLimit, + } + + if !exists { + key := datastore.IncompleteKey("RepoAllowList", nil) + if !dryRun { + if _, err := dsClient.Put(ctx, key, entity); err != nil { + return fmt.Errorf("failed putting entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } else if !existing.matches(item) { + entity.Key = existing.Key + if !dryRun { + if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + return fmt.Errorf("failed updating entity for %s: %w", val, err) + } + } + if verbose { + log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + } + } + + // Delete entries in Datastore that are no longer present in local YAML + for val, existing := range dsEntitiesMap { + if _, exists := localEntriesMap[val]; !exists { + if verbose { + log.Printf("Deleting RepoAllowList entity: val=%s", val) + } + if !dryRun { + if err := dsClient.Delete(ctx, existing.Key); err != nil { + return fmt.Errorf("failed deleting entity for %s: %w", val, err) + } + } + } + } + + if dryRun { + log.Println("[DRY RUN] Sync completed successfully.") + } else { + log.Println("Sync completed successfully.") + } + return nil +} diff --git a/tools/repo-cab-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go similarity index 65% rename from tools/repo-cab-sync/main_test.go rename to tools/repo-allowlist-sync/main_test.go index c7c5a8cca45..cc5e572538d 100644 --- a/tools/repo-cab-sync/main_test.go +++ b/tools/repo-allowlist-sync/main_test.go @@ -74,34 +74,64 @@ func TestParseYAMLEntries(t *testing.T) { yamlContent := []byte(` - type: URL value: " https://github.com/google/osv.dev.git " + consider_all_branches: true + cherrypicks_introduced: true - type: REGEX value: 'github\.com/google/osv-.*' + cherrypicks: true +- type: url + value: "https://github.com/noflags/repo.git" - type: regex value: '[invalid regex' - type: unknown value: "https://github.com/google/osv.dev" - type: url - value: "git@github.com:google/osv.dev.git" + value: "git@github.com:ssh/isnot.supported.git" `) - entries, err := parseYAMLEntries(yamlContent) - if err != nil { - t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) + want := []RepoAllowListEntity{ + // Normalized URL + { + Type: "url", + Value: "github.com/google/osv.dev", + ConsiderAllBranches: true, + CherrypicksIntroduced: true, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, + // Regex type and cherrypicks: true populates all 3 event types + { + Type: "regex", + Value: `github\.com/google/osv-.*`, + ConsiderAllBranches: false, + CherrypicksIntroduced: true, + CherrypicksFixed: true, + CherrypicksLimit: true, + }, + // No flags set (Shouldn't really happen) + { + Type: "url", + Value: "github.com/noflags/repo", + ConsiderAllBranches: false, + CherrypicksIntroduced: false, + CherrypicksFixed: false, + CherrypicksLimit: false, + }, } - // Should normalize types, trim values, and skip invalid regex, unrecognized types, and SSH URLs - if len(entries) != 2 { - t.Fatalf("expected 2 valid entries, got %d: %+v", len(entries), entries) + got, err := parseYAMLEntries(yamlContent) + if err != nil { + t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) } - // First entry should be normalized URL - if entries[0].Type != "url" || entries[0].Value != "github.com/google/osv.dev" { - t.Errorf("entry 0 mismatch: got type=%q val=%q, want type=url val=github.com/google/osv.dev", entries[0].Type, entries[0].Value) + if len(got) != len(want) { + t.Fatalf("expected %d valid entries, got %d: %+v", len(want), len(got), got) } - // Second entry should preserve regex string - if entries[1].Type != "regex" || entries[1].Value != `github\.com/google/osv-.*` { - t.Errorf("entry 1 mismatch: got type=%q val=%q, want type=regex val=github\\.com/google/osv-.*", entries[1].Type, entries[1].Value) + for i, wantEntry := range want { + if got[i] != wantEntry { + t.Errorf("entry %d mismatch:\n got: %+v\nwant: %+v", i, got[i], wantEntry) + } } } diff --git a/tools/repo-allowlist-sync/repo_allowlist.yaml b/tools/repo-allowlist-sync/repo_allowlist.yaml new file mode 100644 index 00000000000..ec257649e33 --- /dev/null +++ b/tools/repo-allowlist-sync/repo_allowlist.yaml @@ -0,0 +1,15 @@ +# Repository allowlist configuration +# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). +# Example: +# - type: url +# value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true +# cherrypicks: true # Shorthand: sets introduced, fixed, and limit to true +# - type: url +# value: "https://github.com/apache/hadoop.git" +# consider_all_branches: true +# cherrypicks: true +# cherrypicks_fixed: false # Specific override for fixed +# - type: regex +# value: 'github\.com/google/osv-.*' +# consider_all_branches: true diff --git a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml b/tools/repo-allowlist-sync/repo_allowlist_test.yaml similarity index 65% rename from tools/repo-cab-sync/repo_cab_allowlist_test.yaml rename to tools/repo-allowlist-sync/repo_allowlist_test.yaml index 4e83defe770..4c6c8bfccd3 100644 --- a/tools/repo-cab-sync/repo_cab_allowlist_test.yaml +++ b/tools/repo-allowlist-sync/repo_allowlist_test.yaml @@ -1,9 +1,13 @@ -# Repository-based consider all branches allowlist +# Repository allowlist configuration # Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). # Example: # - type: url # value: "https://github.com/google/osv.dev.git" +# consider_all_branches: true # - type: regex # value: 'github\.com/google/osv-.*' +# cherrypicks_fixed: true - type: url value: 'https://github.com/apache/hadoop.git' + consider_all_branches: true + cherrypicks: true diff --git a/tools/repo-cab-sync/README.md b/tools/repo-cab-sync/README.md deleted file mode 100644 index de24c7f6431..00000000000 --- a/tools/repo-cab-sync/README.md +++ /dev/null @@ -1,56 +0,0 @@ -# Repo Consider All Branches Allowlist Sync Tool (`repo-cab-sync`) - -`repo-cab-sync` is a Go command-line tool that synchronizes repository "Consider All Branches" (CAB) allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoConsiderAllBranchesAllowList` entities). - -## Overview - -Gitter has the option to enumerate affected commits with `consider_all_branches` enabled or disabled. This tool manages the Datastore allowlist index (`RepoConsiderAllBranchesAllowList`) that controls this behavior on a repository level. - -> [!NOTE] -> If `consider_all_branches` is already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist. - -The tool performs a two-way sync: - -- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities. -- **Delete**: Removes entities from Datastore that are no longer present in the YAML file. - -## Allowlist YAML Format - -The allowlist YAML configuration file accepts a list of entries with `type` and `value` fields: - -```yaml -# Supported entry types: 'url' and 'regex' - -# Exact repository URL match -- type: url - value: "https://github.com/google/osv.dev.git" - -# Regex pattern match (Go RE2 syntax) -- type: regex - value: 'github\.com/google/osv-.*' -``` - -> [!TIP] -> Use single quotes for regex values so you don't have to escape backslashes or other special characters. - -### Normalization and Validation - -- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. -- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. - -## Usage - -Run the tool using `go run`: - -```bash -go run . [flags] -``` - -### Options & Flags - -| Flag | Default | Description | -| ----------- | ------------------------- | ----------------------------------------------------------------- | -| `--file` | `repo_cab_allowlist.yaml` | Path to the input YAML allowlist file | -| `--project` | `oss-vdb-test` | Target GCP Project ID | -| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | -| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-cab-sync/main.go b/tools/repo-cab-sync/main.go deleted file mode 100644 index 7ab771a54c5..00000000000 --- a/tools/repo-cab-sync/main.go +++ /dev/null @@ -1,193 +0,0 @@ -// Package main implements a CLI tool to sync Repo Consider All Branches (CAB) allowlist YAML files to Cloud Datastore. -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net/url" - "os" - "regexp" - "strings" - - "cloud.google.com/go/datastore" - "gopkg.in/yaml.v3" -) - -type RepoCABEntity struct { - Key *datastore.Key `yaml:"-" datastore:"__key__"` - Type string `yaml:"type" datastore:"type"` - Value string `yaml:"value" datastore:"value"` -} - -func main() { - filePath := flag.String("file", "repo_cab_allowlist.yaml", "Path to repo_cab_allowlist YAML file") - project := flag.String("project", "oss-vdb-test", "GCP project ID") - dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") - verbose := flag.Bool("verbose", false, "Display verbose sync operations") - - flag.Parse() - - if *filePath == "" { - log.Fatalf("Error: --file argument is required") - } - - if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { - log.Fatalf("Error syncing repo CAB allowlist: %v", err) - } -} - -func normalizeRepo(repoURL string) string { - // Normalize the repo_url to align with matching logic - // Removes the scheme/protocol, the .git extension, and trailing slashes. - repoURL = strings.TrimSpace(repoURL) - if repoURL == "" { - return "" - } - - if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { - log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) - return "" - } - - parsed, err := url.Parse(repoURL) - if err != nil { - return repoURL - } - normalized := parsed.Host + parsed.Path - normalized = strings.TrimRight(normalized, "/") - normalized = strings.TrimSuffix(normalized, ".git") - - return normalized -} - -func parseYAMLEntries(data []byte) ([]RepoCABEntity, error) { - var parsed []RepoCABEntity - if err := yaml.Unmarshal(data, &parsed); err != nil { - return nil, err - } - - var entries []RepoCABEntity - for _, entry := range parsed { - entry.Type = strings.TrimSpace(strings.ToLower(entry.Type)) - - switch entry.Type { - case "url": - // For repo URLs, we normalize the value before inserting to datastore - entry.Value = normalizeRepo(entry.Value) - if entry.Value == "" { - continue - } - case "regex": - // For regex, we make sure it compiles - if entry.Value == "" { - continue - } - if _, err := regexp.Compile(entry.Value); err != nil { - log.Printf("Warning: Skipping invalid regex pattern %q: %v", entry.Value, err) - continue - } - default: - log.Printf("Warning: Skipping unrecognized entry type %q for value %q", entry.Type, entry.Value) - continue - } - - entries = append(entries, entry) - } - - return entries, nil -} - -func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { - data, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("failed reading file %s: %w", filePath, err) - } - - entries, err := parseYAMLEntries(data) - if err != nil { - return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) - } - - if verbose { - log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) - } - - dsClient, err := datastore.NewClient(ctx, project) - if err != nil { - return fmt.Errorf("failed creating datastore client for project %s: %w", project, err) - } - defer func() { _ = dsClient.Close() }() - - // Get existing Datastore entities - query := datastore.NewQuery("RepoConsiderAllBranchesAllowList") - var dsEntities []RepoCABEntity - if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { - return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) - } - - dsEntitiesMap := make(map[string]RepoCABEntity) - for _, entity := range dsEntities { - dsEntitiesMap[entity.Value] = entity - } - - localEntriesMap := make(map[string]RepoCABEntity) - for _, item := range entries { - localEntriesMap[item.Value] = item - } - - // 1. Put/Upsert entries in local YAML that are not in Datastore or modified - for val, item := range localEntriesMap { - existing, exists := dsEntitiesMap[val] - if !exists { - key := datastore.IncompleteKey("RepoConsiderAllBranchesAllowList", nil) - entity := &RepoCABEntity{ - Type: item.Type, - Value: item.Value, - } - if !dryRun { - if _, err := dsClient.Put(ctx, key, entity); err != nil { - return fmt.Errorf("failed putting entity for %s: %w", val, err) - } - } - if verbose { - log.Printf("Creating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) - } - } else if existing.Type != item.Type { - entity := &RepoCABEntity{ - Type: item.Type, - Value: item.Value, - } - if !dryRun { - if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { - return fmt.Errorf("failed updating entity for %s: %w", val, err) - } - } - if verbose { - log.Printf("Updating RepoConsiderAllBranchesAllowList entity: type=%s val=%s", item.Type, item.Value) - } - } - } - - // 2. Delete entries in Datastore that are no longer in local YAML - for val, existing := range dsEntitiesMap { - if _, exists := localEntriesMap[val]; !exists { - if verbose { - log.Printf("Deleting RepoConsiderAllBranchesAllowList entity: val=%s", val) - } - if !dryRun { - if err := dsClient.Delete(ctx, existing.Key); err != nil { - return fmt.Errorf("failed deleting entity for %s: %w", val, err) - } - } - } - } - - if dryRun { - log.Println("[DRY RUN] Sync completed successfully.") - } else { - log.Println("Sync completed successfully.") - } - return nil -} diff --git a/tools/repo-cab-sync/repo_cab_allowlist.yaml b/tools/repo-cab-sync/repo_cab_allowlist.yaml deleted file mode 100644 index 2b477302dc5..00000000000 --- a/tools/repo-cab-sync/repo_cab_allowlist.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Repository-based consider all branches allowlist -# Supports exact URLs (type: url) and regex (Go RE2 syntax) patterns (type: regex). -# Example: -# - type: url -# value: "https://github.com/google/osv.dev.git" -# - type: regex -# value: 'github\.com/google/osv-.*' From 0c90492507fc8f68c44eca674270b858325f0b8c Mon Sep 17 00:00:00 2001 From: Joey L Date: Fri, 7 Aug 2026 05:41:32 +0000 Subject: [PATCH 06/10] Update readme --- tools/repo-allowlist-sync/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md index ebed96ee8a7..1c8d42e523e 100644 --- a/tools/repo-allowlist-sync/README.md +++ b/tools/repo-allowlist-sync/README.md @@ -16,25 +16,26 @@ The tool performs a two-way sync: ## Allowlist YAML Format -The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields: +The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields. +* Supported `type`s: `url`, `regex` + * `url`: A URL to match against the repository URL. + * `regex`: A regex pattern to match against the repository URL (Go RE2 syntax). +* Supported boolean flags: `consider_all_branches`, `cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`, `cherrypicks` (shorthand for all 3 cherrypick flags) ```yaml -# Supported entry types: 'url' and 'regex' +# Examples -# Shorthand: 'cherrypicks: true' applies to all 3 cherrypick flags (introduced, fixed, limit) - type: url value: "https://github.com/google/osv.dev.git" consider_all_branches: true cherrypicks: true -# Fine-grained control with specific overrides - type: url - value: "https://github.com/apache/hadoop.git" + value: "https://github.com/google/osv.dev.git" consider_all_branches: true cherrypicks: true - cherrypicks_fixed: false # Specific override for fixed + cherrypicks_fixed: false # Overrides cherrypicks: true for fixed event -# Regex pattern match (Go RE2 syntax) - type: regex value: 'github\.com/google/osv-.*' consider_all_branches: true From f3d407e8148ccf9a87c0745a31e6f338f4504ad1 Mon Sep 17 00:00:00 2001 From: Joey L Date: Mon, 10 Aug 2026 06:14:15 +0000 Subject: [PATCH 07/10] address some comments --- tools/repo-allowlist-sync/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go index d8a0a0bf2bb..9603b194323 100644 --- a/tools/repo-allowlist-sync/main.go +++ b/tools/repo-allowlist-sync/main.go @@ -3,6 +3,7 @@ package main import ( "context" + "encoding/base64" "flag" "fmt" "log" @@ -15,6 +16,7 @@ import ( "gopkg.in/yaml.v3" ) +// TODO: Use go model RepoAllowList struct (#5797) // RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore type RepoAllowListEntity struct { Key *datastore.Key `datastore:"__key__"` @@ -39,6 +41,7 @@ type rawYAMLEntry struct { func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { return e.Type == other.Type && + e.Value == other.Value && e.ConsiderAllBranches == other.ConsiderAllBranches && e.CherrypicksIntroduced == other.CherrypicksIntroduced && e.CherrypicksFixed == other.CherrypicksFixed && @@ -200,7 +203,7 @@ func run(ctx context.Context, filePath, project string, dryRun, verbose bool) er } if !exists { - key := datastore.IncompleteKey("RepoAllowList", nil) + key := datastore.NameKey("RepoAllowList", base64.RawURLEncoding.EncodeToString([]byte(val)), nil) if !dryRun { if _, err := dsClient.Put(ctx, key, entity); err != nil { return fmt.Errorf("failed putting entity for %s: %w", val, err) From 1552e2a916af938b89ff2f5cc5a659ba90291ac8 Mon Sep 17 00:00:00 2001 From: Joey L Date: Mon, 10 Aug 2026 07:07:36 +0000 Subject: [PATCH 08/10] yaml v4, strict unmarshalling with field checks, fail early --- tools/repo-allowlist-sync/go.mod | 2 +- tools/repo-allowlist-sync/go.sum | 11 +- tools/repo-allowlist-sync/main.go | 39 ++++---- tools/repo-allowlist-sync/main_test.go | 133 +++++++++++++++++-------- 4 files changed, 114 insertions(+), 71 deletions(-) diff --git a/tools/repo-allowlist-sync/go.mod b/tools/repo-allowlist-sync/go.mod index 941de59e832..d98ec9b482f 100644 --- a/tools/repo-allowlist-sync/go.mod +++ b/tools/repo-allowlist-sync/go.mod @@ -4,7 +4,7 @@ go 1.26.5 require ( cloud.google.com/go/datastore v1.25.0 - gopkg.in/yaml.v3 v3.0.1 + go.yaml.in/yaml/v4 v4.0.0-rc.6 ) require ( diff --git a/tools/repo-allowlist-sync/go.sum b/tools/repo-allowlist-sync/go.sum index 7c948c6563f..2ef435bd78b 100644 --- a/tools/repo-allowlist-sync/go.sum +++ b/tools/repo-allowlist-sync/go.sum @@ -38,16 +38,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -66,6 +60,8 @@ go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRk go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= +go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= @@ -94,8 +90,5 @@ google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go index 9603b194323..422fa47a056 100644 --- a/tools/repo-allowlist-sync/main.go +++ b/tools/repo-allowlist-sync/main.go @@ -13,7 +13,7 @@ import ( "strings" "cloud.google.com/go/datastore" - "gopkg.in/yaml.v3" + "go.yaml.in/yaml/v4" ) // TODO: Use go model RepoAllowList struct (#5797) @@ -66,57 +66,62 @@ func main() { } // normalizeRepo removes the URL scheme, trailing slashes, and .git extensions to standardize repo paths. -func normalizeRepo(repoURL string) string { +func normalizeRepo(repoURL string) (string, error) { repoURL = strings.TrimSpace(repoURL) if repoURL == "" { - return "" + return "", fmt.Errorf("repository URL cannot be empty") } if strings.HasPrefix(repoURL, "git@") || strings.HasPrefix(repoURL, "ssh://") { - log.Printf("Warning: Unsupported SSH URL format %q. Only HTTPS or normalized repository paths are supported.", repoURL) - return "" + return "", fmt.Errorf("unsupported SSH URL format %q: only HTTPS or normalized repository paths are supported", repoURL) } parsed, err := url.Parse(repoURL) if err != nil { - return repoURL + return "", fmt.Errorf("failed to parse URL %q: %w", repoURL, err) } normalized := parsed.Host + parsed.Path normalized = strings.TrimRight(normalized, "/") normalized = strings.TrimSuffix(normalized, ".git") - return normalized + return normalized, nil } // parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. func parseYAMLEntries(data []byte) ([]RepoAllowListEntity, error) { var rawEntries []rawYAMLEntry - if err := yaml.Unmarshal(data, &rawEntries); err != nil { - return nil, err + if err := yaml.Load(data, &rawEntries, yaml.WithKnownFields()); err != nil { + return nil, fmt.Errorf("failed parsing YAML: %w", err) } + seenValues := make(map[string]bool) var entries []RepoAllowListEntity + for _, raw := range rawEntries { raw.Type = strings.TrimSpace(strings.ToLower(raw.Type)) switch raw.Type { case "url": - raw.Value = normalizeRepo(raw.Value) - if raw.Value == "" { - continue + var err error + raw.Value, err = normalizeRepo(raw.Value) + if err != nil { + return nil, fmt.Errorf("invalid repository URL: %w", err) } case "regex": if raw.Value == "" { - continue + return nil, fmt.Errorf("empty regex pattern") } if _, err := regexp.Compile(raw.Value); err != nil { - log.Printf("Warning: Skipping invalid regex pattern %q: %v", raw.Value, err) - continue + return nil, fmt.Errorf("invalid regex pattern %q: %w", raw.Value, err) } default: - log.Printf("Warning: Skipping unrecognized entry type %q for value %q", raw.Type, raw.Value) - continue + return nil, fmt.Errorf("unrecognized entry type %q for value %q", raw.Type, raw.Value) + } + + if seenValues[raw.Value] { + return nil, fmt.Errorf("duplicate allowlist entry value found: %q", raw.Value) } + seenValues[raw.Value] = true // Process cherrypicks flags: "cherrypicks: bool" acts as a shorthand for all 3 event types, // specifying "cherrypicks_" fields overrides that. diff --git a/tools/repo-allowlist-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go index cc5e572538d..c3dae6ff57b 100644 --- a/tools/repo-allowlist-sync/main_test.go +++ b/tools/repo-allowlist-sync/main_test.go @@ -9,68 +9,87 @@ import ( func TestNormalizeRepo(t *testing.T) { tests := []struct { - name string - input string - expected string + name string + input string + expected string + expectError bool }{ { - name: "Empty string", - input: "", - expected: "", + name: "Empty string", + input: "", + expected: "", + expectError: true, }, { - name: "HTTPS URL with .git", - input: "https://github.com/google/osv.dev.git", - expected: "github.com/google/osv.dev", + name: "HTTPS URL with .git", + input: "https://github.com/google/osv.dev.git", + expected: "github.com/google/osv.dev", + expectError: false, }, { - name: "HTTPS URL without .git", - input: "https://github.com/google/osv.dev", - expected: "github.com/google/osv.dev", + name: "HTTPS URL without .git", + input: "https://github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + expectError: false, }, { - name: "URL with trailing slash", - input: "https://github.com/google/osv.dev/", - expected: "github.com/google/osv.dev", + name: "URL with trailing slash", + input: "https://github.com/google/osv.dev/", + expected: "github.com/google/osv.dev", + expectError: false, }, { - name: "No scheme URL", - input: "github.com/google/osv.dev", - expected: "github.com/google/osv.dev", + name: "No scheme URL", + input: "github.com/google/osv.dev", + expected: "github.com/google/osv.dev", + expectError: false, }, { - name: "No scheme with .git", - input: "github.com/google/osv-scanner.git", - expected: "github.com/google/osv-scanner", + name: "No scheme with .git", + input: "github.com/google/osv-scanner.git", + expected: "github.com/google/osv-scanner", + expectError: false, }, { - name: "Whitespace in URL", - input: " https://github.com/google/osv.dev.git ", - expected: "github.com/google/osv.dev", + name: "Whitespace in URL", + input: " https://github.com/google/osv.dev.git ", + expected: "github.com/google/osv.dev", + expectError: false, }, { - name: "SSH URL format git@", - input: "git@github.com:google/osv.dev.git", - expected: "", + name: "SSH URL format git@", + input: "git@github.com:google/osv.dev.git", + expected: "", + expectError: true, }, { - name: "SSH URL format ssh://", - input: "ssh://git@github.com/google/osv.dev.git", - expected: "", + name: "SSH URL format ssh://", + input: "ssh://git@github.com/google/osv.dev.git", + expected: "", + expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := normalizeRepo(tt.input) - if got != tt.expected { - t.Errorf("normalizeRepo(%q) = %q, want %q", tt.input, got, tt.expected) + got, err := normalizeRepo(tt.input) + if tt.expectError { + if err == nil { + t.Errorf("normalizeRepo(%q) expected error, got nil (result: %q)", tt.input, got) + } + } else { + if err != nil { + t.Errorf("normalizeRepo(%q) returned unexpected error: %v", tt.input, err) + } + if got != tt.expected { + t.Errorf("normalizeRepo(%q) = %q, want %q", tt.input, got, tt.expected) + } } }) } } -func TestParseYAMLEntries(t *testing.T) { +func TestParseYAMLEntries_Valid(t *testing.T) { yamlContent := []byte(` - type: URL value: " https://github.com/google/osv.dev.git " @@ -81,16 +100,8 @@ func TestParseYAMLEntries(t *testing.T) { cherrypicks: true - type: url value: "https://github.com/noflags/repo.git" -- type: regex - value: '[invalid regex' -- type: unknown - value: "https://github.com/google/osv.dev" -- type: url - value: "git@github.com:ssh/isnot.supported.git" `) - want := []RepoAllowListEntity{ - // Normalized URL { Type: "url", Value: "github.com/google/osv.dev", @@ -99,7 +110,6 @@ func TestParseYAMLEntries(t *testing.T) { CherrypicksFixed: false, CherrypicksLimit: false, }, - // Regex type and cherrypicks: true populates all 3 event types { Type: "regex", Value: `github\.com/google/osv-.*`, @@ -108,7 +118,6 @@ func TestParseYAMLEntries(t *testing.T) { CherrypicksFixed: true, CherrypicksLimit: true, }, - // No flags set (Shouldn't really happen) { Type: "url", Value: "github.com/noflags/repo", @@ -135,6 +144,42 @@ func TestParseYAMLEntries(t *testing.T) { } } +func TestParseYAMLEntries_Invalid(t *testing.T) { + invalidTests := []struct { + name string + yaml string + }{ + { + name: "Unknown YAML field", + yaml: "- type: url\n value: \"https://github.com/google/osv.dev\"\n unknown_field: true\n", + }, + { + name: "Invalid regex", + yaml: "- type: regex\n value: '[invalid regex'\n", + }, + { + name: "Unrecognized entry type", + yaml: "- type: unknown\n value: \"https://github.com/google/osv.dev\"\n", + }, + { + name: "Unsupported SSH URL", + yaml: "- type: url\n value: \"git@github.com:ssh/isnot.supported.git\"\n", + }, + { + name: "Duplicate values", + yaml: "- type: url\n value: \"https://github.com/google/osv.dev.git\"\n- type: url\n value: \"https://github.com/google/osv.dev\"\n", + }, + } + + for _, tt := range invalidTests { + t.Run(tt.name, func(t *testing.T) { + if _, err := parseYAMLEntries([]byte(tt.yaml)); err == nil { + t.Errorf("parseYAMLEntries expected error for %s, got nil", tt.name) + } + }) + } +} + func TestRun_InvalidFile(t *testing.T) { err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false) if err == nil { From c7d3322835b49463a80dbb671005700e422961e0 Mon Sep 17 00:00:00 2001 From: Joey L Date: Mon, 10 Aug 2026 07:32:34 +0000 Subject: [PATCH 09/10] validate mode --- tools/repo-allowlist-sync/README.md | 15 +-- tools/repo-allowlist-sync/main.go | 163 +++++++++++++++---------- tools/repo-allowlist-sync/main_test.go | 36 +++++- 3 files changed, 136 insertions(+), 78 deletions(-) diff --git a/tools/repo-allowlist-sync/README.md b/tools/repo-allowlist-sync/README.md index 1c8d42e523e..c5b2f86b8ac 100644 --- a/tools/repo-allowlist-sync/README.md +++ b/tools/repo-allowlist-sync/README.md @@ -50,7 +50,7 @@ The allowlist YAML configuration file accepts a list of entries with `type`, `va ### Normalization and Validation - **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`. -- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries are skipped with a warning. +- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries cause validation failure. - **`cherrypicks`**: Acts as a shorthand for setting `cherrypicks_introduced`, `cherrypicks_fixed`, and `cherrypicks_limit` simultaneously. Specific `cherrypicks_` fields override the shorthand value if provided. ## Usage @@ -63,9 +63,10 @@ go run . [flags] ### Options & Flags -| Flag | Default | Description | -| ----------- | --------------------- | ----------------------------------------------------------------- | -| `--file` | `repo_allowlist.yaml` | Path to the input YAML allowlist file | -| `--project` | `oss-vdb-test` | Target GCP Project ID | -| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | -| `--verbose` | `false` | Enables detailed logging of create/update/delete operations | +| Flag | Default | Description | +| ------------ | --------------------- | ----------------------------------------------------------------- | +| `--file` | `repo_allowlist.yaml` | Path to the input YAML allowlist file | +| `--project` | `oss-vdb-test` | Target GCP Project ID | +| `--dry-run` | `true` | When `true`, previews sync operations without modifying Datastore | +| `--validate` | `false` | Validates YAML file and prints summary report without Datastore | +| `--verbose` | `true` | Enables detailed logging of create/update/delete operations | diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go index 422fa47a056..90629b76681 100644 --- a/tools/repo-allowlist-sync/main.go +++ b/tools/repo-allowlist-sync/main.go @@ -4,6 +4,7 @@ package main import ( "context" "encoding/base64" + "errors" "flag" "fmt" "log" @@ -16,6 +17,8 @@ import ( "go.yaml.in/yaml/v4" ) +const repoAllowListKind = "RepoAllowList" + // TODO: Use go model RepoAllowList struct (#5797) // RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore type RepoAllowListEntity struct { @@ -39,6 +42,30 @@ type rawYAMLEntry struct { CherrypicksLimit *bool `yaml:"cherrypicks_limit"` } +func (r rawYAMLEntry) toEntity() RepoAllowListEntity { + intro, fixed, limit := false, false, false + if r.Cherrypicks != nil { + intro, fixed, limit = *r.Cherrypicks, *r.Cherrypicks, *r.Cherrypicks + } + if r.CherrypicksIntroduced != nil { + intro = *r.CherrypicksIntroduced + } + if r.CherrypicksFixed != nil { + fixed = *r.CherrypicksFixed + } + if r.CherrypicksLimit != nil { + limit = *r.CherrypicksLimit + } + return RepoAllowListEntity{ + Type: r.Type, + Value: r.Value, + ConsiderAllBranches: r.ConsiderAllBranches, + CherrypicksIntroduced: intro, + CherrypicksFixed: fixed, + CherrypicksLimit: limit, + } +} + func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { return e.Type == other.Type && e.Value == other.Value && @@ -48,11 +75,16 @@ func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { e.CherrypicksLimit == other.CherrypicksLimit } +func repoAllowListKey(val string) *datastore.Key { + return datastore.NameKey(repoAllowListKind, base64.RawURLEncoding.EncodeToString([]byte(val)), nil) +} + func main() { filePath := flag.String("file", "repo_allowlist.yaml", "Path to repo_allowlist YAML file") project := flag.String("project", "oss-vdb-test", "GCP project ID") dryRun := flag.Bool("dry-run", true, "Perform dry-run without modifying Datastore") - verbose := flag.Bool("verbose", false, "Display verbose sync operations") + validate := flag.Bool("validate", false, "Validate YAML configuration file without modifying Datastore") + verbose := flag.Bool("verbose", true, "Display verbose sync operations") flag.Parse() @@ -60,7 +92,7 @@ func main() { log.Fatalf("Error: --file argument is required") } - if err := run(context.Background(), *filePath, *project, *dryRun, *verbose); err != nil { + if err := run(context.Background(), *filePath, *project, *dryRun, *validate, *verbose); err != nil { log.Fatalf("Error syncing repo allowlist: %v", err) } } @@ -88,86 +120,82 @@ func normalizeRepo(repoURL string) (string, error) { } // parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. -func parseYAMLEntries(data []byte) ([]RepoAllowListEntity, error) { +// If collectAllErrors is true (in validate mode), all entry validation errors are collected and reported together. +// Otherwise, it fails on the first invalid entry. +func parseYAMLEntries(data []byte, collectAllErrors bool) ([]RepoAllowListEntity, error) { var rawEntries []rawYAMLEntry if err := yaml.Load(data, &rawEntries, yaml.WithKnownFields()); err != nil { return nil, fmt.Errorf("failed parsing YAML: %w", err) } - seenValues := make(map[string]bool) - var entries []RepoAllowListEntity + seenValues := make(map[string]int, len(rawEntries)) + entries := make([]RepoAllowListEntity, 0, len(rawEntries)) + var validationErrs []error - for _, raw := range rawEntries { + for i, raw := range rawEntries { + entryNum := i + 1 raw.Type = strings.TrimSpace(strings.ToLower(raw.Type)) + var entryErr error switch raw.Type { case "url": var err error raw.Value, err = normalizeRepo(raw.Value) if err != nil { - return nil, fmt.Errorf("invalid repository URL: %w", err) + entryErr = fmt.Errorf("entry %d: invalid repository URL: %w", entryNum, err) } case "regex": if raw.Value == "" { - return nil, fmt.Errorf("empty regex pattern") - } - if _, err := regexp.Compile(raw.Value); err != nil { - return nil, fmt.Errorf("invalid regex pattern %q: %w", raw.Value, err) + entryErr = fmt.Errorf("entry %d: empty regex pattern", entryNum) + } else if _, err := regexp.Compile(raw.Value); err != nil { + entryErr = fmt.Errorf("entry %d: invalid regex pattern %q: %w", entryNum, raw.Value, err) } default: - return nil, fmt.Errorf("unrecognized entry type %q for value %q", raw.Type, raw.Value) + entryErr = fmt.Errorf("entry %d: unrecognized entry type %q for value %q", entryNum, raw.Type, raw.Value) } - if seenValues[raw.Value] { - return nil, fmt.Errorf("duplicate allowlist entry value found: %q", raw.Value) - } - seenValues[raw.Value] = true - - // Process cherrypicks flags: "cherrypicks: bool" acts as a shorthand for all 3 event types, - // specifying "cherrypicks_" fields overrides that. - intro := false - fixed := false - limit := false - - if raw.Cherrypicks != nil { - intro = *raw.Cherrypicks - fixed = *raw.Cherrypicks - limit = *raw.Cherrypicks - } - if raw.CherrypicksIntroduced != nil { - intro = *raw.CherrypicksIntroduced - } - if raw.CherrypicksFixed != nil { - fixed = *raw.CherrypicksFixed + if entryErr == nil && raw.Value != "" { + if prevIdx, seen := seenValues[raw.Value]; seen { + entryErr = fmt.Errorf("entry %d: duplicate allowlist entry value %q (previously seen at entry %d)", entryNum, raw.Value, prevIdx) + } else { + seenValues[raw.Value] = entryNum + } } - if raw.CherrypicksLimit != nil { - limit = *raw.CherrypicksLimit + + if entryErr != nil { + if !collectAllErrors { + return nil, entryErr + } + validationErrs = append(validationErrs, entryErr) + continue } - entries = append(entries, RepoAllowListEntity{ - Type: raw.Type, - Value: raw.Value, - ConsiderAllBranches: raw.ConsiderAllBranches, - CherrypicksIntroduced: intro, - CherrypicksFixed: fixed, - CherrypicksLimit: limit, - }) + entries = append(entries, raw.toEntity()) + } + + if len(validationErrs) > 0 { + return nil, errors.Join(validationErrs...) } return entries, nil } -func run(ctx context.Context, filePath, project string, dryRun, verbose bool) error { +func run(ctx context.Context, filePath, project string, dryRun, validate, verbose bool) error { data, err := os.ReadFile(filePath) if err != nil { return fmt.Errorf("failed reading file %s: %w", filePath, err) } - entries, err := parseYAMLEntries(data) + entries, err := parseYAMLEntries(data, validate) if err != nil { return fmt.Errorf("failed parsing YAML from %s: %w", filePath, err) } + if validate { + log.Printf("[VALIDATE] YAML file %s is valid. Found %d total entries.", filePath, len(entries)) + return nil + } + if verbose { log.Printf("Loaded %d allowlist entries from %s", len(entries), filePath) } @@ -179,60 +207,60 @@ func run(ctx context.Context, filePath, project string, dryRun, verbose bool) er defer func() { _ = dsClient.Close() }() // Fetch existing Datastore entities - query := datastore.NewQuery("RepoAllowList") + query := datastore.NewQuery(repoAllowListKind) var dsEntities []RepoAllowListEntity if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) } - dsEntitiesMap := make(map[string]RepoAllowListEntity) + dsEntitiesMap := make(map[string]RepoAllowListEntity, len(dsEntities)) for _, entity := range dsEntities { dsEntitiesMap[entity.Value] = entity } - localEntriesMap := make(map[string]RepoAllowListEntity) + localEntriesMap := make(map[string]RepoAllowListEntity, len(entries)) for _, item := range entries { localEntriesMap[item.Value] = item } + var createdCount, updatedCount, deletedCount, unchangedCount int + // Upsert entries in local YAML that are missing from Datastore or modified for val, item := range localEntriesMap { existing, exists := dsEntitiesMap[val] - entity := &RepoAllowListEntity{ - Type: item.Type, - Value: item.Value, - ConsiderAllBranches: item.ConsiderAllBranches, - CherrypicksIntroduced: item.CherrypicksIntroduced, - CherrypicksFixed: item.CherrypicksFixed, - CherrypicksLimit: item.CherrypicksLimit, - } + entity := item if !exists { - key := datastore.NameKey("RepoAllowList", base64.RawURLEncoding.EncodeToString([]byte(val)), nil) + createdCount++ + if verbose { + log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + } + key := repoAllowListKey(val) if !dryRun { - if _, err := dsClient.Put(ctx, key, entity); err != nil { + if _, err := dsClient.Put(ctx, key, &entity); err != nil { return fmt.Errorf("failed putting entity for %s: %w", val, err) } } + } else if !existing.matches(item) { + updatedCount++ if verbose { - log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) } - } else if !existing.matches(item) { entity.Key = existing.Key if !dryRun { - if _, err := dsClient.Put(ctx, existing.Key, entity); err != nil { + if _, err := dsClient.Put(ctx, existing.Key, &entity); err != nil { return fmt.Errorf("failed updating entity for %s: %w", val, err) } } - if verbose { - log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) - } + } else { + unchangedCount++ } } // Delete entries in Datastore that are no longer present in local YAML for val, existing := range dsEntitiesMap { if _, exists := localEntriesMap[val]; !exists { + deletedCount++ if verbose { log.Printf("Deleting RepoAllowList entity: val=%s", val) } @@ -244,10 +272,11 @@ func run(ctx context.Context, filePath, project string, dryRun, verbose bool) er } } + mode := "LIVE" if dryRun { - log.Println("[DRY RUN] Sync completed successfully.") - } else { - log.Println("Sync completed successfully.") + mode = "DRY RUN" } + log.Printf("[%s] Sync completed: %d created, %d updated, %d deleted, %d unchanged.", mode, createdCount, updatedCount, deletedCount, unchangedCount) + return nil } diff --git a/tools/repo-allowlist-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go index c3dae6ff57b..f2791118679 100644 --- a/tools/repo-allowlist-sync/main_test.go +++ b/tools/repo-allowlist-sync/main_test.go @@ -128,7 +128,7 @@ func TestParseYAMLEntries_Valid(t *testing.T) { }, } - got, err := parseYAMLEntries(yamlContent) + got, err := parseYAMLEntries(yamlContent, false) if err != nil { t.Fatalf("parseYAMLEntries returned unexpected error: %v", err) } @@ -173,7 +173,7 @@ func TestParseYAMLEntries_Invalid(t *testing.T) { for _, tt := range invalidTests { t.Run(tt.name, func(t *testing.T) { - if _, err := parseYAMLEntries([]byte(tt.yaml)); err == nil { + if _, err := parseYAMLEntries([]byte(tt.yaml), false); err == nil { t.Errorf("parseYAMLEntries expected error for %s, got nil", tt.name) } }) @@ -181,7 +181,7 @@ func TestParseYAMLEntries_Invalid(t *testing.T) { } func TestRun_InvalidFile(t *testing.T) { - err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false) + err := run(context.Background(), "non_existent_file.yaml", "test-project", true, false, false) if err == nil { t.Error("expected error for non-existent file, got nil") } @@ -192,8 +192,36 @@ func TestRun_InvalidFile(t *testing.T) { t.Fatalf("failed creating bad yaml file: %v", err) } - err = run(context.Background(), badYAMLPath, "test-project", true, false) + err = run(context.Background(), badYAMLPath, "test-project", true, false, false) if err == nil { t.Error("expected error for invalid YAML file, got nil") } } + +func TestRun_Validate(t *testing.T) { + tmpDir := t.TempDir() + + t.Run("Valid YAML", func(t *testing.T) { + validYAML := filepath.Join(tmpDir, "valid.yaml") + if err := os.WriteFile(validYAML, []byte("- type: url\n value: \"https://github.com/google/osv.dev\"\n- type: regex\n value: 'github\\.com/google/.*'\n"), 0644); err != nil { + t.Fatalf("failed creating valid yaml file: %v", err) + } + + // Validate mode (validate=true) should succeed without needing a Datastore connection/project. + if err := run(context.Background(), validYAML, "", true, true, true); err != nil { + t.Errorf("run with validate=true returned unexpected error: %v", err) + } + }) + + t.Run("Invalid YAML", func(t *testing.T) { + invalidYAML := filepath.Join(tmpDir, "invalid.yaml") + if err := os.WriteFile(invalidYAML, []byte("- type: unknown\n value: \"https://github.com/google/osv.dev\"\n- type: regex\n value: '[invalid regex'\n"), 0644); err != nil { + t.Fatalf("failed creating invalid yaml file: %v", err) + } + + err := run(context.Background(), invalidYAML, "", true, true, true) + if err == nil { + t.Error("expected error for invalid YAML file in validate mode, got nil") + } + }) +} From 72a5e8e136f0d6591f7c842a69da4f28003c6791 Mon Sep 17 00:00:00 2001 From: Joey L Date: Wed, 12 Aug 2026 00:19:52 +0000 Subject: [PATCH 10/10] Use go model for datastore entity --- go/osv/models/models.go | 2 + tools/repo-allowlist-sync/go.mod | 56 ++++++- tools/repo-allowlist-sync/go.sum | 202 +++++++++++++++++++++++-- tools/repo-allowlist-sync/main.go | 79 +++++----- tools/repo-allowlist-sync/main_test.go | 4 +- 5 files changed, 280 insertions(+), 63 deletions(-) diff --git a/go/osv/models/models.go b/go/osv/models/models.go index d81202ab0d9..43766eaf935 100644 --- a/go/osv/models/models.go +++ b/go/osv/models/models.go @@ -38,6 +38,8 @@ type Severity = datastore.Severity type ListedVulnerability = datastore.ListedVulnerability +type RepoAllowList = datastore.RepoAllowList + type ImportFindings int const ( diff --git a/tools/repo-allowlist-sync/go.mod b/tools/repo-allowlist-sync/go.mod index d98ec9b482f..8a4df190e90 100644 --- a/tools/repo-allowlist-sync/go.mod +++ b/tools/repo-allowlist-sync/go.mod @@ -4,38 +4,82 @@ go 1.26.5 require ( cloud.google.com/go/datastore v1.25.0 + github.com/google/osv.dev/go v0.0.0 go.yaml.in/yaml/v4 v4.0.0-rc.6 ) +replace github.com/google/osv.dev/go => ../../go + require ( + cel.dev/expr v0.25.1 // indirect + charm.land/lipgloss/v2 v2.0.5 // indirect cloud.google.com/go v0.123.0 // indirect cloud.google.com/go/auth v0.20.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.11.0 // indirect + cloud.google.com/go/monitoring v1.30.0 // indirect + cloud.google.com/go/pubsub/v2 v2.6.1 // indirect + cloud.google.com/go/storage v1.64.0 // indirect + cloud.google.com/go/trace v1.16.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.34.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/colorprofile v0.4.3 // indirect + github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect + github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/term v0.2.2 // indirect + github.com/charmbracelet/x/termios v0.1.1 // indirect + github.com/charmbracelet/x/windows v0.2.2 // indirect + github.com/clipperhouse/displaywidth v0.11.0 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/osv-scalibr v0.4.5 // indirect github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect github.com/googleapis/gax-go/v2 v2.23.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect + github.com/tidwall/gjson v1.19.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect go.opentelemetry.io/otel v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/sdk v1.44.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect golang.org/x/crypto v0.54.0 // indirect - golang.org/x/net v0.56.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/api v0.287.1 // indirect - google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect + google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect google.golang.org/grpc v1.82.1 // indirect - google.golang.org/protobuf v1.36.11 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/dnaeon/go-vcr.v4 v4.0.7 // indirect ) diff --git a/tools/repo-allowlist-sync/go.sum b/tools/repo-allowlist-sync/go.sum index 2ef435bd78b..07c45e56b64 100644 --- a/tools/repo-allowlist-sync/go.sum +++ b/tools/repo-allowlist-sync/go.sum @@ -1,3 +1,8 @@ +cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= +cel.dev/expr v0.25.1/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= +charm.land/lipgloss/v2 v2.0.5 h1:kbNxgeeUOYv5J0YdpxFjfvf3dFvqH8Aci4zB6xqFtrY= +charm.land/lipgloss/v2 v2.0.5/go.mod h1:9oqhxt4yxIMe6q5A4kHr44DremZk7J9UNh74GlWa5nc= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= @@ -8,87 +13,260 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.25.0 h1:zUjMnCLCcRZVDSdQIXsbnNCl1SVRNw5Jm0J77gPaPKs= cloud.google.com/go/datastore v1.25.0/go.mod h1:jvJVNe+S2nHVIndV1H/B4s9K3MLsTMqOKlxSrzHTxB4= +cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= +cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= +cloud.google.com/go/longrunning v1.2.0 h1:WjYH3YHBGCxGJP9M4dWGHBfXr/cFIjMkNgWcJj7/iMM= +cloud.google.com/go/longrunning v1.2.0/go.mod h1:5KMQALFGOCtFoi2xSOA1u3H7WKlhmckgiyFw7+LGQp0= +cloud.google.com/go/monitoring v1.30.0 h1:r/d+JUbyKmJ8b07iznuKfzVzrIXTWxHQ3lBRm3x2LlY= +cloud.google.com/go/monitoring v1.30.0/go.mod h1:htlUR0QWVMrjFzZmN4LGnMAve9xB/eduwjmINxVZ8RM= +cloud.google.com/go/pubsub/v2 v2.6.1 h1:jX6gnC4n8BgYx6MOYICgbbaXZpr1vKeNOE3Bn17P5zg= +cloud.google.com/go/pubsub/v2 v2.6.1/go.mod h1:1y2lZnKfUFPZz0PU4YmXyk4lA11+xmYA42zbC32RkxQ= +cloud.google.com/go/storage v1.64.0 h1:KLpxI/oX9LxeRsNqn877d2WyeT3ryiEwnGt8pwcSPZg= +cloud.google.com/go/storage v1.64.0/go.mod h1:lWyAtwvDZHdL3k68WVKbESP6bmWaV23ZJJ/JEVw/ZaQ= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 h1:rIkQfkCOVKc1OiRCNcSDD8ml5RJlZbH/Xsq7lbpynwc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0/go.mod h1:RD2SsorTmYhF6HkTmDw7KmPYQk8OBYwTkuasChwv7R4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 h1:jLdiS1vO+XJFyDSWRHBx56r4s/NNtcl5J6KyCcWUX/w= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0/go.mod h1:8lmpHY+1VRoteiOwyrQMDt1YGXOrFKCz+1wJW7n3ODY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.34.0 h1:yqXDBvwS4iEl0+xQZqVg9Lz3iWt7GvXbyHF3QhOhqjg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.34.0/go.mod h1:kB72fvJ9MMfrc8DXTsd/+sSnZOPdIYHbKgk/pTKGKm0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0 h1:IBF8BbhKJkMsON/eY+LMu3aF3XMiotCb9KvkUmEkOJo= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.58.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 h1:SBZzZCiPmDrUV7NSCWY54OnKikO/oTydPCvyEyYaDDE= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= +github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= +github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 h1:OqDqxQZliC7C8adA7KjelW3OjtAxREfeHkNcd66wpeI= +github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318/go.mod h1:Y6kE2GzHfkyQQVCSL9r2hwokSrIlHGzZG+71+wDYSZI= +github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= +github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= +github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI= +github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8JawjaNZY= +github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= +github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= +github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= +github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/osv-scalibr v0.4.5 h1:fiJWZg0jXKzFmJiYKs/BhIzUMYUGs0HT2oUZOoKSL+Q= +github.com/google/osv-scalibr v0.4.5/go.mod h1:cNGl//rZ1OcOiFkLXY5DNrhFN7JKMGf4ieQrENUfEZw= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.23.0 h1:Tchl7qkvE7Ip3y+ztvNufYFvkfqTe7NfLTYGIdJRLuE= github.com/googleapis/gax-go/v2 v2.23.0/go.mod h1:rBQKOVJCdb8IFEzg+FCwlt1LP/xMDGuqUXhUG+XMXEg= +github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= +github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec h1:A92d74F0MP8hOiPXRnUnuWaluaf3G3sGJfMEcjkZfjA= +github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec/go.mod h1:IrUa4QzZUi03J3WXDzZYXVawYipHownNfqqZrqeGXfg= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/spiffe/go-spiffe/v2 v2.6.0 h1:l+DolpxNWYgruGQVV0xsfeya3CsC7m8iBzDnMpsbLuo= +github.com/spiffe/go-spiffe/v2 v2.6.0/go.mod h1:gm2SeUoMZEtpnzPNs2Csc0D/gX33k1xIx7lEzqblHEs= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0 h1:yI1/OhfEPy7J9eoa6Sj051C7n5dvpj0QX8g4sRchg04= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.67.0/go.mod h1:NoUCKYWK+3ecatC4HjkRktREheMeEtrXoQxrqYFeHSc= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0 h1:NmLfL734pJhM0JKaYd2Y28+nY9dPRWYAAbxhRCrKXPw= +go.opentelemetry.io/contrib/detectors/gcp v1.44.0/go.mod h1:tNAsgd8avTGke1+MndXlU5Cru4PQ9Ai/cCNWQv/ZJ/s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 h1:2yEATaop1/a1I4psnSLgWVPLWwCzkqWakgJy7xTDVy0= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0/go.mod h1:D7J12YRapIekYyPWgGPlA/23pRmpSEZC5xJC/TTLI9U= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0 h1:hqxVTu/GtBF+vJ8d1fzW7fRxZFvgoDjWcxwwCaFDYpU= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.44.0/go.mod h1:z5fVEF4X5v0ESvlJqBrrFlBVoj5EQuefZpzsu7R+x5Q= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= -golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.287.1 h1:LiyJx32VU3cwQfLchn/513qKhc25hq0pEANYJoWNnnI= google.golang.org/api v0.287.1/go.mod h1:lM2kYRzYUCBY91P9h6VF1PYmvhxii3O5hji37qRvIcY= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= -google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 h1:YJjbgu+dkp5kUJLfpMyCLfBIWZb/FcJyuLeo1gVBOuo= +google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94/go.mod h1:RRHjglSYABVCWpQ7USCpdfhcd9t4PkajvVwyynZizTc= google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 h1:jQ9p21COKWjP3VwuFrNRiiOTMh3mPpN45R7SLrH/HUU= google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7/go.mod h1:KqHwBx2upmfa1XSi1WuRvC+2VGCLtooKkfmyvRbUmqA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/dnaeon/go-vcr.v4 v4.0.7 h1:Mq/RF+mq3QwtEunJSsoTbYPt3elSAmdJhAxrEaqr88I= +gopkg.in/dnaeon/go-vcr.v4 v4.0.7/go.mod h1:cRwV/njsN/D8qNJu4NAXWswz6b4OUh3rMIu4SObbLBg= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/tools/repo-allowlist-sync/main.go b/tools/repo-allowlist-sync/main.go index 90629b76681..9ddc0366a2c 100644 --- a/tools/repo-allowlist-sync/main.go +++ b/tools/repo-allowlist-sync/main.go @@ -14,23 +14,12 @@ import ( "strings" "cloud.google.com/go/datastore" + "github.com/google/osv.dev/go/osv/models" "go.yaml.in/yaml/v4" ) const repoAllowListKind = "RepoAllowList" -// TODO: Use go model RepoAllowList struct (#5797) -// RepoAllowListEntity represents a repository allowlist entity stored in Cloud Datastore -type RepoAllowListEntity struct { - Key *datastore.Key `datastore:"__key__"` - Type string `datastore:"type"` - Value string `datastore:"value"` - ConsiderAllBranches bool `datastore:"consider_all_branches"` - CherrypicksIntroduced bool `datastore:"cherrypicks_introduced"` - CherrypicksFixed bool `datastore:"cherrypicks_fixed"` - CherrypicksLimit bool `datastore:"cherrypicks_limit"` -} - // rawYAMLEntry represents an unmarshaled entry from the YAML file, including optional shorthand field type rawYAMLEntry struct { Type string `yaml:"type"` @@ -42,7 +31,7 @@ type rawYAMLEntry struct { CherrypicksLimit *bool `yaml:"cherrypicks_limit"` } -func (r rawYAMLEntry) toEntity() RepoAllowListEntity { +func (r rawYAMLEntry) toEntity() models.RepoAllowList { intro, fixed, limit := false, false, false if r.Cherrypicks != nil { intro, fixed, limit = *r.Cherrypicks, *r.Cherrypicks, *r.Cherrypicks @@ -56,7 +45,7 @@ func (r rawYAMLEntry) toEntity() RepoAllowListEntity { if r.CherrypicksLimit != nil { limit = *r.CherrypicksLimit } - return RepoAllowListEntity{ + return models.RepoAllowList{ Type: r.Type, Value: r.Value, ConsiderAllBranches: r.ConsiderAllBranches, @@ -66,13 +55,13 @@ func (r rawYAMLEntry) toEntity() RepoAllowListEntity { } } -func (e RepoAllowListEntity) matches(other RepoAllowListEntity) bool { - return e.Type == other.Type && - e.Value == other.Value && - e.ConsiderAllBranches == other.ConsiderAllBranches && - e.CherrypicksIntroduced == other.CherrypicksIntroduced && - e.CherrypicksFixed == other.CherrypicksFixed && - e.CherrypicksLimit == other.CherrypicksLimit +func entriesMatch(a, b models.RepoAllowList) bool { + return a.Type == b.Type && + a.Value == b.Value && + a.ConsiderAllBranches == b.ConsiderAllBranches && + a.CherrypicksIntroduced == b.CherrypicksIntroduced && + a.CherrypicksFixed == b.CherrypicksFixed && + a.CherrypicksLimit == b.CherrypicksLimit } func repoAllowListKey(val string) *datastore.Key { @@ -122,14 +111,14 @@ func normalizeRepo(repoURL string) (string, error) { // parseYAMLEntries parses and validates allowlist YAML content, expanding shorthand fields and normalizing values. // If collectAllErrors is true (in validate mode), all entry validation errors are collected and reported together. // Otherwise, it fails on the first invalid entry. -func parseYAMLEntries(data []byte, collectAllErrors bool) ([]RepoAllowListEntity, error) { +func parseYAMLEntries(data []byte, collectAllErrors bool) ([]models.RepoAllowList, error) { var rawEntries []rawYAMLEntry if err := yaml.Load(data, &rawEntries, yaml.WithKnownFields()); err != nil { return nil, fmt.Errorf("failed parsing YAML: %w", err) } seenValues := make(map[string]int, len(rawEntries)) - entries := make([]RepoAllowListEntity, 0, len(rawEntries)) + entries := make([]models.RepoAllowList, 0, len(rawEntries)) var validationErrs []error for i, raw := range rawEntries { @@ -208,47 +197,49 @@ func run(ctx context.Context, filePath, project string, dryRun, validate, verbos // Fetch existing Datastore entities query := datastore.NewQuery(repoAllowListKind) - var dsEntities []RepoAllowListEntity - if _, err := dsClient.GetAll(ctx, query, &dsEntities); err != nil { + var dsEntities []models.RepoAllowList + keys, err := dsClient.GetAll(ctx, query, &dsEntities) + if err != nil { return fmt.Errorf("failed fetching existing allowlist entities from datastore: %w", err) } - dsEntitiesMap := make(map[string]RepoAllowListEntity, len(dsEntities)) - for _, entity := range dsEntities { - dsEntitiesMap[entity.Value] = entity + remoteEntities := make(map[string]models.RepoAllowList, len(dsEntities)) + remoteKeys := make(map[string]*datastore.Key, len(dsEntities)) + for i, entity := range dsEntities { + remoteEntities[entity.Value] = entity + remoteKeys[entity.Value] = keys[i] } - localEntriesMap := make(map[string]RepoAllowListEntity, len(entries)) + localEntities := make(map[string]models.RepoAllowList, len(entries)) for _, item := range entries { - localEntriesMap[item.Value] = item + localEntities[item.Value] = item } var createdCount, updatedCount, deletedCount, unchangedCount int // Upsert entries in local YAML that are missing from Datastore or modified - for val, item := range localEntriesMap { - existing, exists := dsEntitiesMap[val] - entity := item + for val, local := range localEntities { + existing, exists := remoteEntities[val] if !exists { createdCount++ if verbose { - log.Printf("Creating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + log.Printf("Creating RepoAllowList entity: type=%s val=%s", local.Type, local.Value) } key := repoAllowListKey(val) if !dryRun { - if _, err := dsClient.Put(ctx, key, &entity); err != nil { - return fmt.Errorf("failed putting entity for %s: %w", val, err) + if _, err := dsClient.Put(ctx, key, &local); err != nil { + return fmt.Errorf("failed creating entity for %s: %w", val, err) } } - } else if !existing.matches(item) { + } else if !entriesMatch(existing, local) { updatedCount++ if verbose { - log.Printf("Updating RepoAllowList entity: type=%s val=%s", item.Type, item.Value) + log.Printf("Updating RepoAllowList entity: type=%s val=%s", local.Type, local.Value) } - entity.Key = existing.Key + key := remoteKeys[val] if !dryRun { - if _, err := dsClient.Put(ctx, existing.Key, &entity); err != nil { + if _, err := dsClient.Put(ctx, key, &local); err != nil { return fmt.Errorf("failed updating entity for %s: %w", val, err) } } @@ -257,15 +248,15 @@ func run(ctx context.Context, filePath, project string, dryRun, validate, verbos } } - // Delete entries in Datastore that are no longer present in local YAML - for val, existing := range dsEntitiesMap { - if _, exists := localEntriesMap[val]; !exists { + // Delete remote entries in Datastore that are no longer present in local YAML + for val := range remoteEntities { + if _, exists := localEntities[val]; !exists { deletedCount++ if verbose { log.Printf("Deleting RepoAllowList entity: val=%s", val) } if !dryRun { - if err := dsClient.Delete(ctx, existing.Key); err != nil { + if err := dsClient.Delete(ctx, remoteKeys[val]); err != nil { return fmt.Errorf("failed deleting entity for %s: %w", val, err) } } diff --git a/tools/repo-allowlist-sync/main_test.go b/tools/repo-allowlist-sync/main_test.go index f2791118679..78eaa124809 100644 --- a/tools/repo-allowlist-sync/main_test.go +++ b/tools/repo-allowlist-sync/main_test.go @@ -5,6 +5,8 @@ import ( "os" "path/filepath" "testing" + + "github.com/google/osv.dev/go/osv/models" ) func TestNormalizeRepo(t *testing.T) { @@ -101,7 +103,7 @@ func TestParseYAMLEntries_Valid(t *testing.T) { - type: url value: "https://github.com/noflags/repo.git" `) - want := []RepoAllowListEntity{ + want := []models.RepoAllowList{ { Type: "url", Value: "github.com/google/osv.dev",