fix(project): filter duplicate items in update batches

internal/apps/project/item_filter.go

@@ -0,0 +1,44 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2025 linux.do
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package project
+
+// filterNewItems removes values that already exist or occur earlier in items,
+// while preserving the order of the first occurrence in items.
+func filterNewItems(items, existingItems []string) []string {
+	existingSet := make(map[string]struct{}, len(existingItems)+len(items))
+	for _, item := range existingItems {
+		existingSet[item] = struct{}{}
+	}
+
+	filtered := make([]string, 0, len(items))
+	for _, item := range items {
+		if _, exists := existingSet[item]; exists {
+			continue
+		}
+		existingSet[item] = struct{}{}
+		filtered = append(filtered, item)
+	}
+	return filtered
+}

internal/apps/project/item_filter_test.go

@@ -0,0 +1,66 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2025 linux.do
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package project
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFilterNewItems(t *testing.T) {
+	tests := []struct {
+		name          string
+		items         []string
+		existingItems []string
+		want          []string
+	}{
+		{
+			name:          "filters persisted and request duplicates",
+			items:         []string{"already-there", "new", "new", "another", "already-there"},
+			existingItems: []string{"already-there"},
+			want:          []string{"new", "another"},
+		},
+		{
+			name:  "preserves first occurrence order",
+			items: []string{"b", "a", "b", "c", "a"},
+			want:  []string{"b", "a", "c"},
+		},
+		{
+			name:          "handles empty input",
+			items:         nil,
+			existingItems: []string{"existing"},
+			want:          []string{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := filterNewItems(tt.items, tt.existingItems)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Fatalf("filterNewItems() = %#v, want %#v", got, tt.want)
+			}
+		})
+	}
+}

internal/apps/project/models.go

@@ -313,19 +313,7 @@ func (p *Project) CreateItemsWithFilter(ctx context.Context, tx *gorm.DB, items
 			return err
 		}
 
-		// Create a set of existing items for O(1) lookup
-		existingSet := make(map[string]bool)
-		for _, item := range existingItems {
-			existingSet[item] = true
-		}
-
-		// Filter out duplicates
-		filteredItems = make([]string, 0, len(items))
-		for _, item := range items {
-			if !existingSet[item] {
-				filteredItems = append(filteredItems, item)
-			}
-		}
+		filteredItems = filterNewItems(items, existingItems)
 	}
 
 	// Use the original CreateItems method with filtered items
@@ -350,21 +338,7 @@ func (p *Project) GetFilteredItemsCount(ctx context.Context, tx *gorm.DB, items
 		return 0, err
 	}
 
-	// Create a set of existing items for O(1) lookup
-	existingSet := make(map[string]bool)
-	for _, item := range existingItems {
-		existingSet[item] = true
-	}
-
-	// Count unique items
-	uniqueCount := int64(0)
-	for _, item := range items {
-		if !existingSet[item] {
-			uniqueCount++
-		}
-	}
-
-	return uniqueCount, nil
+	return int64(len(filterNewItems(items, existingItems))), nil
 }
 
 func (p *Project) PrepareReceive(ctx context.Context, userName string) (uint64, error) {