feat: sync tool for repo-based git enumeration flags allowlist

go/osv/models/models.go

@@ -38,6 +38,8 @@ type Severity = datastore.Severity
 
 type ListedVulnerability = datastore.ListedVulnerability
 
+type RepoAllowList = datastore.RepoAllowList
+
 type ImportFindings int
 
 const (

tools/repo-allowlist-sync/README.md

@@ -0,0 +1,72 @@
+# Repository Allowlist Sync Tool (`repo-allowlist-sync`)
+
+`repo-allowlist-sync` is a Go command-line tool that synchronizes repository allowlist configuration files (`.yaml`) to Cloud Datastore (`RepoAllowList` entities).
+
+## Overview
+
+Gitter has options to enumerate affected commits with `consider_all_branches` and cherrypick detection options (`cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`). This tool manages the Datastore allowlist index (`RepoAllowList`) that controls these behaviors on a repository level.
+
+> [!NOTE]
+> If feature flags are already enabled at the `SourceRepository` level, you do not need to add the repository to this allowlist.
+
+The tool performs a two-way sync:
+
+- **Upsert**: Adds new allowlist entries from the local YAML file to Datastore, or updates modified entities.
+- **Delete**: Removes entities from Datastore that are no longer present in the YAML file.
+
+## Allowlist YAML Format
+
+The allowlist YAML configuration file accepts a list of entries with `type`, `value`, and boolean feature flag fields.
+* Supported `type`s: `url`, `regex`
+  * `url`: A URL to match against the repository URL.
+  * `regex`: A regex pattern to match against the repository URL (Go RE2 syntax).
+* Supported boolean flags: `consider_all_branches`, `cherrypicks_introduced`, `cherrypicks_fixed`, `cherrypicks_limit`, `cherrypicks` (shorthand for all 3 cherrypick flags)
+
+```yaml
+# Examples
+
+- type: url
+  value: "https://github.com/google/osv.dev.git"
+  consider_all_branches: true
+  cherrypicks: true
+
+- type: url
+  value: "https://github.com/google/osv.dev.git"
+  consider_all_branches: true
+  cherrypicks: true
+  cherrypicks_fixed: false            # Overrides cherrypicks: true for fixed event
+
+- type: regex
+  value: 'github\.com/google/osv-.*'
+  consider_all_branches: true
+  cherrypicks_introduced: true
+  cherrypicks_fixed: true
+  cherrypicks_limit: true
+```
+
+> [!TIP]
+> Use single quotes for regex values so you don't have to escape backslashes or other special characters.
+
+### Normalization and Validation
+
+- **`type: url`**: Repository URLs are automatically normalized before being saved to Datastore (removing protocol scheme, `.git` extension, and trailing slashes). For example, `https://github.com/google/osv.dev.git` is normalized to `github.com/google/osv.dev`.
+- **`type: regex`**: Regular expressions are compiled and validated against Go's RE2 standard syntax. Invalid regex entries cause validation failure.
+- **`cherrypicks`**: Acts as a shorthand for setting `cherrypicks_introduced`, `cherrypicks_fixed`, and `cherrypicks_limit` simultaneously. Specific `cherrypicks_<type>` fields override the shorthand value if provided.
+
+## Usage
+
+Run the tool using `go run`:
+
+```bash
+go run . [flags]
+```
+
+### Options & Flags
+
+| Flag         | Default               | Description                                                       |
+| ------------ | --------------------- | ----------------------------------------------------------------- |
+| `--file`     | `repo_allowlist.yaml` | Path to the input YAML allowlist file                             |
+| `--project`  | `oss-vdb-test`        | Target GCP Project ID                                             |
+| `--dry-run`  | `true`                | When `true`, previews sync operations without modifying Datastore |
+| `--validate` | `false`               | Validates YAML file and prints summary report without Datastore   |
+| `--verbose`  | `true`                | Enables detailed logging of create/update/delete operations       |

tools/repo-allowlist-sync/go.mod

@@ -0,0 +1,85 @@
+module github.com/google/osv.dev/tools/repo-allowlist-sync
+
+go 1.26.5
+
+require (
+	cloud.google.com/go/datastore v1.25.0
+	github.com/google/osv.dev/go v0.0.0
+	go.yaml.in/yaml/v4 v4.0.0-rc.6
+)
+
+replace github.com/google/osv.dev/go => ../../go
+
+require (
+	cel.dev/expr v0.25.1 // indirect
+	charm.land/lipgloss/v2 v2.0.5 // indirect
+	cloud.google.com/go v0.123.0 // indirect
+	cloud.google.com/go/auth v0.20.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect
+	cloud.google.com/go/compute/metadata v0.9.0 // indirect
+	cloud.google.com/go/iam v1.11.0 // indirect
+	cloud.google.com/go/monitoring v1.30.0 // indirect
+	cloud.google.com/go/pubsub/v2 v2.6.1 // indirect
+	cloud.google.com/go/storage v1.64.0 // indirect
+	cloud.google.com/go/trace v1.16.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.32.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.57.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace v1.34.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.58.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/colorprofile v0.4.3 // indirect
+	github.com/charmbracelet/ultraviolet v0.0.0-20251205161215-1948445e3318 // indirect
+	github.com/charmbracelet/x/ansi v0.11.7 // indirect
+	github.com/charmbracelet/x/term v0.2.2 // indirect
+	github.com/charmbracelet/x/termios v0.1.1 // indirect
+	github.com/charmbracelet/x/windows v0.2.2 // indirect
+	github.com/clipperhouse/displaywidth v0.11.0 // indirect
+	github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/osv-scalibr v0.4.5 // indirect
+	github.com/google/s2a-go v0.1.9 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect
+	github.com/googleapis/gax-go/v2 v2.23.0 // indirect
+	github.com/lucasb-eyer/go-colorful v1.4.0 // indirect
+	github.com/mattn/go-runewidth v0.0.23 // indirect
+	github.com/muesli/cancelreader v0.2.2 // indirect
+	github.com/ossf/osv-schema/bindings/go v0.0.0-20260806060209-f3f826310aec // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
+	github.com/tidwall/gjson v1.19.0 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.69.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect
+	go.opentelemetry.io/otel v1.44.0 // indirect
+	go.opentelemetry.io/otel/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.44.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.44.0 // indirect
+	go.opentelemetry.io/otel/trace v1.44.0 // indirect
+	golang.org/x/crypto v0.54.0 // indirect
+	golang.org/x/net v0.57.0 // indirect
+	golang.org/x/oauth2 v0.36.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
+	golang.org/x/sys v0.47.0 // indirect
+	golang.org/x/text v0.40.0 // indirect
+	golang.org/x/time v0.15.0 // indirect
+	google.golang.org/api v0.287.1 // indirect
+	google.golang.org/genproto v0.0.0-20260519071638-aa98bba5eb94 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20260630182238-925bb5da69e7 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 // indirect
+	google.golang.org/grpc v1.82.1 // indirect
+	google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect
+	gopkg.in/dnaeon/go-vcr.v4 v4.0.7 // indirect
+)