controlplane: store file hashes when building the flavor version

Makefile

@@ -33,6 +33,7 @@ generate: sqlc genproto mocks
 fmt:
 	@find . -type f -name '*.go' \
        -not -path './vendor/*' \
+       -not -path './.idea/*' \
        -not -name '*.pb.go' \
        -exec gofmt -w {} +
 

controlplane/chunk/flavor.go

@@ -22,7 +22,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"maps"
+	"os"
 	"path/filepath"
 	"slices"
 	"sort"
@@ -36,6 +38,7 @@ import (
 	"github.com/spacechunks/explorer/controlplane/job"
 	"github.com/spacechunks/explorer/internal/file"
 	"github.com/spacechunks/explorer/internal/resource"
+	"github.com/spacechunks/explorer/internal/tarhelper"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -136,9 +139,23 @@ func (s *svc) CreateFlavorVersion(
 
 	prevVersion, err := s.repo.LatestFlavorVersion(ctx, flavorID)
 	if err != nil {
-		return resource.FlavorVersion{},
-			resource.FlavorVersionDiff{},
-			fmt.Errorf("latest flavor version file hashes: %w", err)
+		// super, super, ugly, but as of right now i don't want to refactor
+		// (it returns ErrNotFound, if this is the first flavor version)
+		if errors.Is(err, apierrs.ErrNotFound) {
+			prevVersion.FilesUploaded = true
+		} else {
+			return resource.FlavorVersion{},
+				resource.FlavorVersionDiff{},
+				fmt.Errorf("latest flavor version file hashes: %w", err)
+		}
+	}
+
+	// we do not allow creating a new flavor version when the previous one did not have their
+	// files uploaded, because we depend on the uploaded files, when building the image later.
+	// this is because we only upload what changed between versions. if the previous changes
+	// are not uploaded to s3, the build_image job will fail.
+	if !prevVersion.FilesUploaded {
+		return resource.FlavorVersion{}, resource.FlavorVersionDiff{}, apierrs.ErrPreviousFilesNotUploaded
 	}
 
 	newContentTree, err := file.HashTree(version.FileHashes)
@@ -212,19 +229,6 @@ func (s *svc) CreateFlavorVersion(
 	sortByPath(added)
 	sortByPath(removed)
 
-	changes := make([]file.Hash, 0, len(changed)+len(added))
-	changes = append(changes, changed...)
-	changes = append(changes, added...)
-	sortByPath(changes)
-
-	all := make([]file.Hash, 0, len(unchanged)+len(changes))
-	all = append(all, changes...)
-	all = append(all, unchanged...)
-
-	sortByPath(all)
-
-	version.FileHashes = all
-
 	created, err := s.repo.CreateFlavorVersion(ctx, flavorID, version, prevVersion.ID)
 	if err != nil {
 		return resource.FlavorVersion{}, resource.FlavorVersionDiff{}, fmt.Errorf("create flavor version: %w", err)
@@ -300,6 +304,15 @@ func (s *svc) BuildFlavorVersion(ctx context.Context, versionID string) error {
 			return apierrs.ErrFlavorFilesNotUploaded
 		}
 
+		hashes, err := s.computeFileHashes(ctx, versionID)
+		if err != nil {
+			return fmt.Errorf("compute file hashes: %w", err)
+		}
+
+		if err := s.repo.AddFlavorVersionFileHashes(ctx, versionID, hashes); err != nil {
+			return fmt.Errorf("add flavor version hashes: %w", err)
+		}
+
 		if err := s.repo.MarkFlavorVersionFilesUploaded(ctx, versionID); err != nil {
 			return fmt.Errorf("mark files: %w", err)
 		}
@@ -412,3 +425,70 @@ func (s *svc) GetFlavor(ctx context.Context, id string) (resource.Flavor, error)
 
 	return f, nil
 }
+
+func (s *svc) computeFileHashes(ctx context.Context, versionID string) ([]file.Hash, error) {
+	dir, err := os.MkdirTemp("", fmt.Sprintf("changeset-%s-*", versionID))
+	if err != nil {
+		return nil, fmt.Errorf("create tmp dir: %w", err)
+	}
+
+	set, err := os.Create(filepath.Join(dir, "changeset.tar.gz"))
+	if err != nil {
+		return nil, fmt.Errorf("tmp file: %w", err)
+	}
+
+	defer set.Close()
+
+	defer func() {
+		if err := os.RemoveAll(dir); err != nil {
+			s.logger.Error("failed to remove temp dir", "err", err)
+		}
+	}()
+
+	if err := s.s3Store.WriteTo(ctx, blob.ChangeSetKey(versionID), set); err != nil {
+		return nil, fmt.Errorf("write: %w", err)
+	}
+
+	if _, err := set.Seek(0, io.SeekStart); err != nil {
+		return nil, fmt.Errorf("seek: %w", err)
+	}
+
+	paths, err := tarhelper.Untar(set, dir)
+	if err != nil {
+		return nil, fmt.Errorf("untar: %w", err)
+	}
+
+	hashes := make([]file.Hash, 0, len(paths))
+
+	for _, p := range paths {
+		if err := func() error {
+			f, err := os.Open(p)
+			if err != nil {
+				return fmt.Errorf("open: %w", err)
+			}
+
+			defer f.Close()
+
+			hash, err := file.ComputeHashStr(f)
+			if err != nil {
+				return fmt.Errorf("compute hash: %w", err)
+			}
+
+			serverRootPath, err := filepath.Rel(dir, p)
+			if err != nil {
+				return fmt.Errorf("server root path: %w", err)
+			}
+
+			hashes = append(hashes, file.Hash{
+				Path: serverRootPath,
+				Hash: hash,
+			})
+
+			return nil
+		}(); err != nil {
+			return nil, err
+		}
+	}
+
+	return hashes, nil
+}

controlplane/chunk/flavor_internal_test.go

@@ -0,0 +1,77 @@
+/*
+ Explorer Platform, a platform for hosting and discovering Minecraft servers.
+ Copyright (C) 2024 Yannic Rieger <oss@76k.io>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+*/
+
+package chunk
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/spacechunks/explorer/controlplane/blob"
+	"github.com/spacechunks/explorer/internal/file"
+	"github.com/spacechunks/explorer/internal/mock"
+	"github.com/spacechunks/explorer/test"
+	mocky "github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComputeFileHashes(t *testing.T) {
+	var (
+		ctx     = context.Background()
+		tarData = test.CreateTarGz(t, map[string]string{
+			"server.properties":   "bla",
+			"plugins/config.yaml": "lol",
+		})
+		mockStore = mock.NewMockBlobS3Store(t)
+		versionID = "version-id"
+		expected  = []file.Hash{
+			{
+				Path: "server.properties",
+				Hash: "038d12a21e489bb2",
+			},
+			{
+				Path: "plugins/config.yaml",
+				Hash: "7b75e34aa5423334",
+			},
+		}
+	)
+
+	mockStore.EXPECT().
+		WriteTo(mocky.Anything, blob.ChangeSetKey(versionID), mocky.Anything).
+		RunAndReturn(func(ctx context.Context, key string, w io.Writer) error {
+			_, err := w.Write(tarData)
+			require.NoError(t, err)
+			return nil
+		})
+
+	s := svc{
+		s3Store: mockStore,
+	}
+
+	actual, err := s.computeFileHashes(ctx, versionID)
+	require.NoError(t, err)
+
+	file.SortHashes(expected)
+	file.SortHashes(actual)
+
+	if d := cmp.Diff(expected, actual); d != "" {
+		t.Fatalf("mismatch (-want +got):\n%s", d)
+	}
+}

controlplane/chunk/flavor_test.go

@@ -165,6 +165,10 @@ func TestCreateFlavorVersion(t *testing.T) {
 		return version
 	}
 
+	prevVersionFilesUploaded := fixture.FlavorVersion(func(tmp *resource.FlavorVersion) {
+		tmp.FilesUploaded = true
+	})
+
 	tests := []struct {
 		name         string
 		prevVersion  resource.FlavorVersion
@@ -182,7 +186,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 	}{
 		{
 			name:        "works",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion: fixture.FlavorVersion(func(v *resource.FlavorVersion) {
 				v.Version = "v2"
 				v.FileHashes = []file.Hash{
@@ -261,7 +265,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "cleans paths",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion:  uncleanPathVersion(),
 			expected:    ptr.Pointer(cleanedPathVersion()),
 			expectedDiff: resource.FlavorVersionDiff{
@@ -326,7 +330,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "rejects relative traversal paths",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion: fixture.FlavorVersion(func(v *resource.FlavorVersion) {
 				v.Version = "v2"
 				v.FileHashes = []file.Hash{
@@ -390,7 +394,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "rejects absolute paths",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion: fixture.FlavorVersion(func(v *resource.FlavorVersion) {
 				v.Version = "v2"
 				v.FileHashes = []file.Hash{
@@ -445,7 +449,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "version hash mismatch",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion: fixture.FlavorVersion(func(v *resource.FlavorVersion) {
 				v.Hash = "some-not-matching-hash"
 				v.FileHashes = []file.Hash{
@@ -501,7 +505,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "version already exists",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion:  fixture.FlavorVersion(),
 			prep: func(
 				repo *mock.MockChunkRepository,
@@ -528,7 +532,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "minecraft version unsupported",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion:  fixture.FlavorVersion(),
 			prep: func(
 				repo *mock.MockChunkRepository,
@@ -559,7 +563,7 @@ func TestCreateFlavorVersion(t *testing.T) {
 		},
 		{
 			name:        "flavor deleted",
-			prevVersion: fixture.FlavorVersion(),
+			prevVersion: prevVersionFilesUploaded,
 			newVersion:  fixture.FlavorVersion(),
 			prep: func(
 				repo *mock.MockChunkRepository,

controlplane/chunk/repository.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/spacechunks/explorer/internal/file"
 	"github.com/spacechunks/explorer/internal/resource"
 )
 
@@ -60,6 +61,7 @@ type Repository interface {
 	MarkFlavorDeleted(ctx context.Context, id string) error
 	FlavorByID(ctx context.Context, id string) (resource.Flavor, error)
 	ChunkByFlavorID(ctx context.Context, flavorID string) (resource.Chunk, error)
+	AddFlavorVersionFileHashes(ctx context.Context, flavorVersionID string, hashes []file.Hash) error
 }
 
 type ArchiveRepository interface {

controlplane/errors/error.go

@@ -75,8 +75,11 @@ var (
 	ErrMinecraftVersionNotSupported = New(codes.FailedPrecondition, "minecraft version not found")
 	ErrHashMismatch                 = New(codes.FailedPrecondition, "hash does not match")
 	ErrFlavorFilesNotUploaded       = New(codes.FailedPrecondition, "flavor files have not been uploaded")
-	ErrFlavorFilesUploaded          = New(codes.AlreadyExists, "flavor files have already been uploaded")
-	ErrChangeSetTarballTooBig       = New(codes.InvalidArgument, "tarball size exceeds maximum allowed")
+	ErrPreviousFilesNotUploaded     = New(
+		codes.FailedPrecondition, "files of the last flavor version have not been uploaded",
+	)
+	ErrFlavorFilesUploaded    = New(codes.AlreadyExists, "flavor files have already been uploaded")
+	ErrChangeSetTarballTooBig = New(codes.InvalidArgument, "tarball size exceeds maximum allowed")
 )
 
 /*