From 5fcb248a8c43165dad78ecd58204a19cbb4ae6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BF=9E=E5=8F=AF=20=C2=B7=20YU=20KE?= <1798456934@qq.com> Date: Tue, 1 Sep 2026 00:17:20 +0800 Subject: [PATCH] fix(project): filter duplicate items in update batches Co-Authored-By: EvoX --- internal/apps/project/item_filter.go | 44 +++++++++++++++ internal/apps/project/item_filter_test.go | 66 +++++++++++++++++++++++ internal/apps/project/models.go | 30 +---------- 3 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 internal/apps/project/item_filter.go create mode 100644 internal/apps/project/item_filter_test.go diff --git a/internal/apps/project/item_filter.go b/internal/apps/project/item_filter.go new file mode 100644 index 0000000..e6e4b4c --- /dev/null +++ b/internal/apps/project/item_filter.go @@ -0,0 +1,44 @@ +/* + * MIT License + * + * Copyright (c) 2025 linux.do + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package project + +// filterNewItems removes values that already exist or occur earlier in items, +// while preserving the order of the first occurrence in items. +func filterNewItems(items, existingItems []string) []string { + existingSet := make(map[string]struct{}, len(existingItems)+len(items)) + for _, item := range existingItems { + existingSet[item] = struct{}{} + } + + filtered := make([]string, 0, len(items)) + for _, item := range items { + if _, exists := existingSet[item]; exists { + continue + } + existingSet[item] = struct{}{} + filtered = append(filtered, item) + } + return filtered +} diff --git a/internal/apps/project/item_filter_test.go b/internal/apps/project/item_filter_test.go new file mode 100644 index 0000000..e3b274d --- /dev/null +++ b/internal/apps/project/item_filter_test.go @@ -0,0 +1,66 @@ +/* + * MIT License + * + * Copyright (c) 2025 linux.do + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package project + +import ( + "reflect" + "testing" +) + +func TestFilterNewItems(t *testing.T) { + tests := []struct { + name string + items []string + existingItems []string + want []string + }{ + { + name: "filters persisted and request duplicates", + items: []string{"already-there", "new", "new", "another", "already-there"}, + existingItems: []string{"already-there"}, + want: []string{"new", "another"}, + }, + { + name: "preserves first occurrence order", + items: []string{"b", "a", "b", "c", "a"}, + want: []string{"b", "a", "c"}, + }, + { + name: "handles empty input", + items: nil, + existingItems: []string{"existing"}, + want: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := filterNewItems(tt.items, tt.existingItems) + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("filterNewItems() = %#v, want %#v", got, tt.want) + } + }) + } +} diff --git a/internal/apps/project/models.go b/internal/apps/project/models.go index d4f730d..bca31cf 100644 --- a/internal/apps/project/models.go +++ b/internal/apps/project/models.go @@ -313,19 +313,7 @@ func (p *Project) CreateItemsWithFilter(ctx context.Context, tx *gorm.DB, items return err } - // Create a set of existing items for O(1) lookup - existingSet := make(map[string]bool) - for _, item := range existingItems { - existingSet[item] = true - } - - // Filter out duplicates - filteredItems = make([]string, 0, len(items)) - for _, item := range items { - if !existingSet[item] { - filteredItems = append(filteredItems, item) - } - } + filteredItems = filterNewItems(items, existingItems) } // Use the original CreateItems method with filtered items @@ -350,21 +338,7 @@ func (p *Project) GetFilteredItemsCount(ctx context.Context, tx *gorm.DB, items return 0, err } - // Create a set of existing items for O(1) lookup - existingSet := make(map[string]bool) - for _, item := range existingItems { - existingSet[item] = true - } - - // Count unique items - uniqueCount := int64(0) - for _, item := range items { - if !existingSet[item] { - uniqueCount++ - } - } - - return uniqueCount, nil + return int64(len(filterNewItems(items, existingItems))), nil } func (p *Project) PrepareReceive(ctx context.Context, userName string) (uint64, error) {