From b802c63ec3d3746d53eb4a1331bb97f6c32ffece Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Wed, 5 Aug 2026 00:21:26 +0000 Subject: [PATCH 1/4] fix canonical link issue --- vulnfeeds/conversion/common.go | 40 +++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index e2097ce5f2f..44cf9f98ead 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -145,10 +145,32 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri unresolvedRanges := versionRanges var successfulRepos []string + localCanonicalCache := make(map[string]string) + getCanonical := func(repo string) (string, error) { + if canonical, ok := localCanonicalCache[repo]; ok { + return canonical, nil + } + canonical, err := git.FindCanonicalLink(repo, http.DefaultClient, cache) + if err == nil { + localCanonicalCache[repo] = canonical + } + + return canonical, err + } + claimedRepos := make(map[string]bool) for _, vr := range versionRanges { if vr.Range.GetRepo() != "" { - claimedRepos[vr.Range.GetRepo()] = true + canonicalRepo, err := getCanonical(vr.Range.GetRepo()) + if err != nil { + if git.IsRateLimit(err) { + metrics.Outcome = models.Error + return nil, nil, nil + } + claimedRepos[vr.Range.GetRepo()] = true + } else { + claimedRepos[canonicalRepo] = true + } } } @@ -160,7 +182,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri continue } - repo, err := git.FindCanonicalLink(repo, http.DefaultClient, cache) + repo, err := getCanonical(repo) if err != nil { metrics.AddNote("Failed to find canonical link - %s %v", repo, err) if git.IsRateLimit(err) { @@ -184,7 +206,19 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri var stillUnresolvedRanges []models.RangeWithMetadata for _, vr := range unresolvedRanges { - if (vr.Range.GetRepo() != "" && vr.Range.GetRepo() != repo) || (vr.Range.GetRepo() == "" && claimedRepos[repo]) { + vRepo := vr.Range.GetRepo() + if vRepo != "" { + canonicalVRepo, err := getCanonical(vRepo) + if err != nil { + if git.IsRateLimit(err) { + metrics.Outcome = models.Error + return nil, nil, nil + } + } else { + vRepo = canonicalVRepo + } + } + if (vRepo != "" && vRepo != repo) || (vRepo == "" && claimedRepos[repo]) { stillUnresolvedRanges = append(stillUnresolvedRanges, vr) continue } From b075f13a6f19ed8966970b195e41850c36f5e5b2 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Thu, 6 Aug 2026 02:03:43 +0000 Subject: [PATCH 2/4] no local --- vulnfeeds/conversion/common.go | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index 44cf9f98ead..9148db2b881 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -145,23 +145,10 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri unresolvedRanges := versionRanges var successfulRepos []string - localCanonicalCache := make(map[string]string) - getCanonical := func(repo string) (string, error) { - if canonical, ok := localCanonicalCache[repo]; ok { - return canonical, nil - } - canonical, err := git.FindCanonicalLink(repo, http.DefaultClient, cache) - if err == nil { - localCanonicalCache[repo] = canonical - } - - return canonical, err - } - claimedRepos := make(map[string]bool) for _, vr := range versionRanges { if vr.Range.GetRepo() != "" { - canonicalRepo, err := getCanonical(vr.Range.GetRepo()) + canonicalRepo, err := git.FindCanonicalLink(vr.Range.GetRepo(), http.DefaultClient, cache) if err != nil { if git.IsRateLimit(err) { metrics.Outcome = models.Error @@ -182,7 +169,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri continue } - repo, err := getCanonical(repo) + repo, err := git.FindCanonicalLink(repo, http.DefaultClient, cache) if err != nil { metrics.AddNote("Failed to find canonical link - %s %v", repo, err) if git.IsRateLimit(err) { @@ -208,7 +195,7 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri for _, vr := range unresolvedRanges { vRepo := vr.Range.GetRepo() if vRepo != "" { - canonicalVRepo, err := getCanonical(vRepo) + canonicalVRepo, err := git.FindCanonicalLink(vRepo, http.DefaultClient, cache) if err != nil { if git.IsRateLimit(err) { metrics.Outcome = models.Error From 4c51af948a91f5b4746b0e247223801b443b3f17 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Mon, 10 Aug 2026 00:07:53 +0000 Subject: [PATCH 3/4] claim both repos --- vulnfeeds/conversion/common.go | 7 +- vulnfeeds/conversion/common_test.go | 184 ++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 2 deletions(-) diff --git a/vulnfeeds/conversion/common.go b/vulnfeeds/conversion/common.go index 60a49cedc54..082778eb305 100644 --- a/vulnfeeds/conversion/common.go +++ b/vulnfeeds/conversion/common.go @@ -145,18 +145,21 @@ func GitVersionsToCommits(versionRanges []models.RangeWithMetadata, repos []stri unresolvedRanges := versionRanges var successfulRepos []string + // claimedRepos tracks repositories explicitly specified in version ranges. + // This prevents generic version ranges (without a repo) from being processed + // against repositories that are explicitly targeted by other ranges. claimedRepos := make(map[string]bool) for _, vr := range versionRanges { if vr.Range.GetRepo() != "" { + claimedRepos[vr.Range.GetRepo()] = true // Always claim the raw repository URL. canonicalRepo, err := git.FindCanonicalLink(vr.Range.GetRepo(), http.DefaultClient, cache) if err != nil { if git.IsRateLimit(err) { metrics.Outcome = models.Error return nil, nil, nil } - claimedRepos[vr.Range.GetRepo()] = true } else { - claimedRepos[canonicalRepo] = true + claimedRepos[canonicalRepo] = true // Also claim the canonical URL if different. } } } diff --git a/vulnfeeds/conversion/common_test.go b/vulnfeeds/conversion/common_test.go index 162e807ee1d..eb54ff8434c 100644 --- a/vulnfeeds/conversion/common_test.go +++ b/vulnfeeds/conversion/common_test.go @@ -1,9 +1,11 @@ package conversion import ( + "os" "testing" "github.com/google/go-cmp/cmp" + "github.com/google/osv/vulnfeeds/git" "github.com/google/osv/vulnfeeds/models" "github.com/ossf/osv-schema/bindings/go/osvschema" "google.golang.org/protobuf/testing/protocmp" @@ -454,3 +456,185 @@ func TestCreateUnresolvedRanges(t *testing.T) { }) } } + +func TestGitVersionsToCommits_Canonicalization(t *testing.T) { + tests := []struct { + name string + versionRanges []models.RangeWithMetadata + repos []string + canonicalLinks map[string]string + cachedTags map[string]git.RepoTagsMap + wantResolved int + wantUnresolved int + wantSuccessful []string + }{ + { + name: "Range repo is alias, Loop repo is canonical", + versionRanges: []models.RangeWithMetadata{ + { + Range: &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: "http://github.com/alias/repo", + Events: []*osvschema.Event{ + {Introduced: "1.0.0"}, + {Fixed: "1.0.1"}, + }, + }, + }, + }, + repos: []string{"https://github.com/canonical/repo"}, + canonicalLinks: map[string]string{ + "http://github.com/alias/repo": "https://github.com/canonical/repo", + "https://github.com/canonical/repo": "https://github.com/canonical/repo", + }, + cachedTags: map[string]git.RepoTagsMap{ + "https://github.com/canonical/repo": { + NormalizedTag: map[string]git.NormalizedTag{ + "1-0-0": {OriginalTag: "v1.0.0", Commit: "100commit"}, + "1-0-1": {OriginalTag: "v1.0.1", Commit: "101commit"}, + }, + }, + }, + wantResolved: 1, + wantUnresolved: 0, + wantSuccessful: []string{"https://github.com/canonical/repo"}, + }, + { + name: "Range repo is canonical, Loop repo is alias", + versionRanges: []models.RangeWithMetadata{ + { + Range: &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: "https://github.com/canonical/repo", + Events: []*osvschema.Event{ + {Introduced: "1.0.0"}, + {Fixed: "1.0.1"}, + }, + }, + }, + }, + repos: []string{"http://github.com/alias/repo"}, + canonicalLinks: map[string]string{ + "http://github.com/alias/repo": "https://github.com/canonical/repo", + "https://github.com/canonical/repo": "https://github.com/canonical/repo", + }, + cachedTags: map[string]git.RepoTagsMap{ + "https://github.com/canonical/repo": { + NormalizedTag: map[string]git.NormalizedTag{ + "1-0-0": {OriginalTag: "v1.0.0", Commit: "100commit"}, + "1-0-1": {OriginalTag: "v1.0.1", Commit: "101commit"}, + }, + }, + }, + wantResolved: 1, + wantUnresolved: 0, + wantSuccessful: []string{"https://github.com/canonical/repo"}, + }, + { + name: "Range without repo, Loop repo is alias", + versionRanges: []models.RangeWithMetadata{ + { + Range: &osvschema.Range{ + Type: osvschema.Range_GIT, + Events: []*osvschema.Event{ + {Introduced: "1.0.0"}, + {Fixed: "1.0.1"}, + }, + }, + }, + }, + repos: []string{"http://github.com/alias/repo"}, + canonicalLinks: map[string]string{ + "http://github.com/alias/repo": "https://github.com/canonical/repo", + "https://github.com/canonical/repo": "https://github.com/canonical/repo", + }, + cachedTags: map[string]git.RepoTagsMap{ + "https://github.com/canonical/repo": { + NormalizedTag: map[string]git.NormalizedTag{ + "1-0-0": {OriginalTag: "v1.0.0", Commit: "100commit"}, + "1-0-1": {OriginalTag: "v1.0.1", Commit: "101commit"}, + }, + }, + }, + wantResolved: 1, + wantUnresolved: 0, + wantSuccessful: []string{"https://github.com/canonical/repo"}, + }, + { + name: "Range without repo skipped if repo claimed by another range", + versionRanges: []models.RangeWithMetadata{ + { + Range: &osvschema.Range{ + Type: osvschema.Range_GIT, + Repo: "http://github.com/alias/repo", // Claims the repo + Events: []*osvschema.Event{ + {Introduced: "1.0.0"}, + {Fixed: "1.0.1"}, + }, + }, + }, + { + Range: &osvschema.Range{ + Type: osvschema.Range_GIT, // No repo + Events: []*osvschema.Event{ + {Introduced: "2.0.0"}, + {Fixed: "2.0.1"}, + }, + }, + }, + }, + repos: []string{"https://github.com/canonical/repo"}, + canonicalLinks: map[string]string{ + "http://github.com/alias/repo": "https://github.com/canonical/repo", + "https://github.com/canonical/repo": "https://github.com/canonical/repo", + }, + cachedTags: map[string]git.RepoTagsMap{ + "https://github.com/canonical/repo": { + NormalizedTag: map[string]git.NormalizedTag{ + "1-0-0": {OriginalTag: "v1.0.0", Commit: "100commit"}, + "1-0-1": {OriginalTag: "v1.0.1", Commit: "101commit"}, + "2-0-0": {OriginalTag: "v2.0.0", Commit: "200commit"}, + "2-0-1": {OriginalTag: "v2.0.1", Commit: "201commit"}, + }, + }, + }, + wantResolved: 1, // Only the first range should be resolved by this repo + wantUnresolved: 1, // The second range should remain unresolved + wantSuccessful: []string{"https://github.com/canonical/repo"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + oldRedisHost := os.Getenv("REDISHOST") + os.Unsetenv("REDISHOST") + defer func() { + if oldRedisHost != "" { + os.Setenv("REDISHOST", oldRedisHost) + } + }() + + cache := git.NewRepoTagsCache() + for k, v := range tt.canonicalLinks { + cache.SetCanonicalLink(k, v) + } + for k, v := range tt.cachedTags { + cache.Set(k, v) + } + + metrics := &models.ConversionMetrics{} + gotResolved, gotUnresolved, gotSuccessful := GitVersionsToCommits(tt.versionRanges, tt.repos, metrics, cache) + + if len(gotResolved) != tt.wantResolved { + t.Errorf("GitVersionsToCommits() gotResolved count = %v, want %v", len(gotResolved), tt.wantResolved) + } + if len(gotUnresolved) != tt.wantUnresolved { + t.Errorf("GitVersionsToCommits() gotUnresolved count = %v, want %v", len(gotUnresolved), tt.wantUnresolved) + } + if diff := cmp.Diff(tt.wantSuccessful, gotSuccessful); diff != "" { + t.Errorf("GitVersionsToCommits() gotSuccessful mismatch (-want +got):\n%s", diff) + } + }) + } +} + From 3e165dd264c756f9c133a5ab4cb9662582998ec3 Mon Sep 17 00:00:00 2001 From: Jess Lowe Date: Mon, 10 Aug 2026 00:21:48 +0000 Subject: [PATCH 4/4] fix lint --- vulnfeeds/conversion/common_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vulnfeeds/conversion/common_test.go b/vulnfeeds/conversion/common_test.go index eb54ff8434c..81b490091d4 100644 --- a/vulnfeeds/conversion/common_test.go +++ b/vulnfeeds/conversion/common_test.go @@ -610,7 +610,7 @@ func TestGitVersionsToCommits_Canonicalization(t *testing.T) { os.Unsetenv("REDISHOST") defer func() { if oldRedisHost != "" { - os.Setenv("REDISHOST", oldRedisHost) + t.Setenv("REDISHOST", oldRedisHost) } }() @@ -637,4 +637,3 @@ func TestGitVersionsToCommits_Canonicalization(t *testing.T) { }) } } -