From 39b7b16a57ea5551b2f391bae25f1e4dd5a4ca47 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Wed, 26 Mar 2025 21:27:09 +0100 Subject: [PATCH 01/11] Add app command structure and error handling for Giant Swarm applications --- README.md | 150 +++++++- cmd/app/bootstrap/command.go | 61 ++++ cmd/app/bootstrap/error.go | 39 +++ cmd/app/bootstrap/flag.go | 67 ++++ cmd/app/bootstrap/runner.go | 480 ++++++++++++++++++++++++++ cmd/app/command.go | 72 ++++ cmd/app/error.go | 21 ++ cmd/app/flag.go | 16 + cmd/app/runner.go | 42 +++ cmd/command.go | 16 + go.mod | 2 + go.sum | 4 + pkg/githubclient/client_repository.go | 44 +++ 13 files changed, 997 insertions(+), 17 deletions(-) create mode 100644 cmd/app/bootstrap/command.go create mode 100644 cmd/app/bootstrap/error.go create mode 100644 cmd/app/bootstrap/flag.go create mode 100644 cmd/app/bootstrap/runner.go create mode 100644 cmd/app/command.go create mode 100644 cmd/app/error.go create mode 100644 cmd/app/flag.go create mode 100644 cmd/app/runner.go diff --git a/README.md b/README.md index 2cee97c7f..508b9a096 100644 --- a/README.md +++ b/README.md @@ -2,36 +2,152 @@ # devctl -Command line tool for the daily development business at Giant Swarm. +`devctl` is a command-line tool designed to streamline development workflows at Giant Swarm. It provides various commands to help manage repositories, generate files, and bootstrap applications. -## Quick start +## Installation -### Installation +```bash +go install github.com/giantswarm/devctl/v7@latest +``` + +## Features + +### App Management (`devctl app`) + +The `app` command provides tools for working with Giant Swarm app repositories. + +#### Bootstrap Command (`devctl app bootstrap`) + +Creates a new app repository from a template with the following features: + +- Creates a new repository from the `template-app` template +- Configures sync methods (vendir or kustomize) +- Sets up patch methods (script or kustomize) +- Configures CI/CD automatically +- Creates a pull request with the initial setup + +```bash +devctl app bootstrap \ + --name my-app \ + --patch-method script \ + --sync-method vendir \ + --team myteam \ + --upstream-chart helm/upstream \ + --upstream-repo https://github.com/org/repo \ + --github-token-envvar GITHUB_TOKEN +``` + +Options: +- `--name`: Name of the app (required) +- `--patch-method`: Method to patch upstream changes (script or kustomize) +- `--sync-method`: Method to sync upstream changes (vendir or kustomize) +- `--team`: Team responsible for the app +- `--upstream-chart`: Path to the upstream chart +- `--upstream-repo`: URL of the upstream repository +- `--github-token-envvar`: Name of environment variable containing GitHub token +- `--dry-run`: Only print what would be done + +### Repository Management (`devctl repo`) + +The `repo` command helps manage GitHub repositories. + +#### Setup Command (`devctl repo setup`) + +Configures repository settings according to Giant Swarm standards: + +- Sets up branch protection rules +- Configures repository settings +- Sets up team permissions +- Configures CI/CD workflows + +```bash +devctl repo setup org/repo-name +``` + +### Code Generation (`devctl gen`) + +The `gen` command provides tools for generating various files: + +- Workflows +- Makefile +- License +- And more -Install the [latest release](https://github.com/giantswarm/devctl/releases/latest). Please do not use `go install`. +```bash +devctl gen workflow +devctl gen makefile +devctl gen license +``` + +### Release Management (`devctl release`) + +Helps manage releases in Giant Swarm repositories: + +- Creates release branches +- Updates changelog +- Creates GitHub releases + +```bash +devctl release create +``` + +### Replace Command (`devctl replace`) + +Helps replace content across multiple files: + +```bash +devctl replace old-text new-text +``` -### Configuration +### Shell Completion -Most commands require credentials for GitHub write access to be available. Make sure you have the environment variable +Provides shell completion for various shells: -```nohighlight -GITHUB_TOKEN +```bash +# Bash +devctl completion bash > ~/.bash_completion.d/devctl + +# Zsh +devctl completion zsh > "${fpath[1]}/_devctl" + +# Fish +devctl completion fish > ~/.config/fish/completions/devctl.fish ``` -set with a valid [personal access token](https://github.com/settings/tokens) as the value. +## Development + +### Requirements + +- Go 1.21 or later +- Git +- GitHub account with appropriate permissions + +### Building from Source -### Usage +```bash +git clone https://github.com/giantswarm/devctl.git +cd devctl +go build +``` + +### Running Tests -Please check `devctl --help` for available commands and options. +```bash +go test ./... +``` -Also see the [docs](docs/) folder for more details on some commands. +### Debug Mode -### Updating +Set `LOG_LEVEL=debug` to see detailed output: -```nohighlight -devctl version update +```bash +LOG_LEVEL=debug devctl app bootstrap ... ``` -### Development +## Contributing + +Please check our [contributing guidelines](CONTRIBUTING.md) for details on how to contribute to this project. + +## License -While running locally during development you may get some errors relating to `no matching files found` for some of the templates. If you do run `make generate-go` to generate these template files before running. +devctl is licensed under the [Apache 2.0 License](LICENSE). diff --git a/cmd/app/bootstrap/command.go b/cmd/app/bootstrap/command.go new file mode 100644 index 000000000..9e9362e60 --- /dev/null +++ b/cmd/app/bootstrap/command.go @@ -0,0 +1,61 @@ +package bootstrap + +import ( + "io" + "os" + + "github.com/giantswarm/microerror" + "github.com/giantswarm/micrologger" + "github.com/spf13/cobra" +) + +const ( + name = "bootstrap" + description = "Bootstrap a new Giant Swarm app repository from an upstream Helm chart" + example = ` devctl app bootstrap \ + --name n8n \ + --upstream-repo https://github.com/n8n-io/n8n \ + --upstream-chart charts/n8n \ + --team planeteers \ + --sync-method vendir \ + --patch-method script` +) + +type Config struct { + Logger micrologger.Logger + Stderr io.Writer + Stdout io.Writer +} + +func New(config Config) (*cobra.Command, error) { + if config.Logger == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config) + } + if config.Stderr == nil { + config.Stderr = os.Stderr + } + if config.Stdout == nil { + config.Stdout = os.Stdout + } + + f := &flag{} + + r := &runner{ + flag: f, + logger: config.Logger, + stderr: config.Stderr, + stdout: config.Stdout, + } + + c := &cobra.Command{ + Use: name, + Short: description, + Long: description, + Example: example, + RunE: r.Run, + } + + f.Init(c) + + return c, nil +} diff --git a/cmd/app/bootstrap/error.go b/cmd/app/bootstrap/error.go new file mode 100644 index 000000000..3fd7e0a5d --- /dev/null +++ b/cmd/app/bootstrap/error.go @@ -0,0 +1,39 @@ +package bootstrap + +import "github.com/giantswarm/microerror" + +var invalidConfigError = µerror.Error{ + Kind: "invalidConfigError", +} + +// IsInvalidConfig asserts invalidConfigError. +func IsInvalidConfig(err error) bool { + return microerror.Cause(err) == invalidConfigError +} + +var invalidFlagError = µerror.Error{ + Kind: "invalidFlagError", +} + +// IsInvalidFlag asserts invalidFlagError. +func IsInvalidFlag(err error) bool { + return microerror.Cause(err) == invalidFlagError +} + +var envVarNotFoundError = µerror.Error{ + Kind: "envVarNotFoundError", +} + +// IsEnvVarNotFound asserts envVarNotFoundError. +func IsEnvVarNotFound(err error) bool { + return microerror.Cause(err) == envVarNotFoundError +} + +var executionFailedError = µerror.Error{ + Kind: "executionFailedError", +} + +// IsExecutionFailed asserts executionFailedError. +func IsExecutionFailed(err error) bool { + return microerror.Cause(err) == executionFailedError +} diff --git a/cmd/app/bootstrap/flag.go b/cmd/app/bootstrap/flag.go new file mode 100644 index 000000000..1b5f2de84 --- /dev/null +++ b/cmd/app/bootstrap/flag.go @@ -0,0 +1,67 @@ +package bootstrap + +import ( + "github.com/giantswarm/microerror" + "github.com/spf13/cobra" +) + +const ( + flagName = "name" + flagUpstreamRepo = "upstream-repo" + flagUpstreamChart = "upstream-chart" + flagTeam = "team" + flagSyncMethod = "sync-method" + flagPatchMethod = "patch-method" + flagGithubToken = "github-token-envvar" + flagDryRun = "dry-run" +) + +type flag struct { + Name string + UpstreamRepo string + UpstreamChart string + Team string + SyncMethod string + PatchMethod string + GithubToken string + DryRun bool +} + +func (f *flag) Init(cmd *cobra.Command) { + cmd.Flags().StringVar(&f.Name, flagName, "", "Name of the app to bootstrap") + cmd.Flags().StringVar(&f.UpstreamRepo, flagUpstreamRepo, "", "URL of the upstream repository containing the Helm chart") + cmd.Flags().StringVar(&f.UpstreamChart, flagUpstreamChart, "", "Path to the Helm chart in the upstream repository") + cmd.Flags().StringVar(&f.Team, flagTeam, "", "Team responsible for the app") + cmd.Flags().StringVar(&f.SyncMethod, flagSyncMethod, "vendir", "Method to sync upstream chart (vendir or kustomize)") + cmd.Flags().StringVar(&f.PatchMethod, flagPatchMethod, "script", "Method to patch upstream chart (script or kustomize)") + cmd.Flags().StringVar(&f.GithubToken, flagGithubToken, "GITHUB_TOKEN", "Name of environment variable containing GitHub token") + cmd.Flags().BoolVar(&f.DryRun, flagDryRun, false, "If set, only print what would be done") + + cmd.MarkFlagRequired(flagName) + cmd.MarkFlagRequired(flagUpstreamRepo) + cmd.MarkFlagRequired(flagUpstreamChart) + cmd.MarkFlagRequired(flagTeam) +} + +func (f *flag) Validate() error { + if f.Name == "" { + return microerror.Maskf(invalidFlagError, "--%s must not be empty", flagName) + } + if f.UpstreamRepo == "" { + return microerror.Maskf(invalidFlagError, "--%s must not be empty", flagUpstreamRepo) + } + if f.UpstreamChart == "" { + return microerror.Maskf(invalidFlagError, "--%s must not be empty", flagUpstreamChart) + } + if f.Team == "" { + return microerror.Maskf(invalidFlagError, "--%s must not be empty", flagTeam) + } + if f.SyncMethod != "vendir" && f.SyncMethod != "kustomize" { + return microerror.Maskf(invalidFlagError, "--%s must be either 'vendir' or 'kustomize'", flagSyncMethod) + } + if f.PatchMethod != "script" && f.PatchMethod != "kustomize" { + return microerror.Maskf(invalidFlagError, "--%s must be either 'script' or 'kustomize'", flagPatchMethod) + } + + return nil +} diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go new file mode 100644 index 000000000..de0868a06 --- /dev/null +++ b/cmd/app/bootstrap/runner.go @@ -0,0 +1,480 @@ +package bootstrap + +import ( + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/briandowns/spinner" + "github.com/giantswarm/microerror" + "github.com/giantswarm/micrologger" + "github.com/google/go-github/v70/github" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/giantswarm/devctl/v7/pkg/githubclient" +) + +type runner struct { + flag *flag + logger micrologger.Logger + stdout io.Writer + stderr io.Writer +} + +func (r *runner) Run(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + err := r.flag.Validate() + if err != nil { + return microerror.Mask(err) + } + + err = r.run(ctx, cmd, args) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) error { + s := spinner.New(spinner.CharSets[14], 100*time.Millisecond) + s.Writer = r.stdout + + // 1. Create repository from app-template + s.Suffix = " Creating repository from template..." + s.Start() + err := r.createRepository(ctx, r.flag.Name, "giantswarm", "template-app") + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Repository created from template") + + // Wait for repository to be fully created and initialized + s.Suffix = " Waiting for repository to be initialized..." + s.Start() + time.Sleep(10 * time.Second) + s.Stop() + + // 2. Clone repository locally + s.Suffix = " Cloning repository..." + s.Start() + repoPath, err := r.cloneRepository(ctx) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Repository cloned locally") + + // 3. Configure sync method (vendir/kustomize) + s.Suffix = fmt.Sprintf(" Configuring sync method (%s)...", r.flag.SyncMethod) + s.Start() + err = r.configureSyncMethod(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Sync method configured") + + // 4. Configure patch method (script/kustomize) + s.Suffix = fmt.Sprintf(" Configuring patch method (%s)...", r.flag.PatchMethod) + s.Start() + err = r.configurePatchMethod(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Patch method configured") + + // 5. Replace placeholders + s.Suffix = " Replacing placeholders..." + s.Start() + err = r.replacePlaceholders(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Placeholders replaced") + + // 6. Setup CI/CD + s.Suffix = " Setting up CI/CD..." + s.Start() + err = r.setupCICD(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ CI/CD setup complete") + + // 7. Initial commit and push + s.Suffix = " Creating pull request..." + s.Start() + err = r.commitAndPush(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Pull request created") + + fmt.Fprintf(r.stdout, "\n✨ Successfully bootstrapped app repository %s\n", r.flag.Name) + + return nil +} + +func (r *runner) createRepository(ctx context.Context, name string, owner string, templateRepo string) error { + token := os.Getenv(r.flag.GithubToken) + if token == "" { + return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) + } + + // Create a logger that only outputs in debug mode + logger := logrus.New() + if os.Getenv("LOG_LEVEL") == "debug" { + logger.SetOutput(r.stdout) + } else { + logger.SetOutput(io.Discard) + } + + config := githubclient.Config{ + Logger: logger, + AccessToken: token, + DryRun: r.flag.DryRun, + } + + client, err := githubclient.New(config) + if err != nil { + return microerror.Mask(err) + } + + repoName := fmt.Sprintf("%s-app", name) + repo := &github.Repository{ + Name: github.String(repoName), + Private: github.Bool(false), + Description: github.String(fmt.Sprintf("Helm chart for %s", name)), + } + + _, err = client.CreateFromTemplate(ctx, owner, templateRepo, owner, repo) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) cloneRepository(ctx context.Context) (string, error) { + // Clone repository locally + repoPath := filepath.Join(os.TempDir(), fmt.Sprintf("%s-app", r.flag.Name)) + repoURL := fmt.Sprintf("git@github.com:giantswarm/%s-app.git", r.flag.Name) + + // Remove existing directory if it exists + _ = os.RemoveAll(repoPath) + + // Wait for repository to be populated with template content + maxRetries := 10 + for i := 0; i < maxRetries; i++ { + err := r.execCommand(ctx, "", "git", "clone", repoURL, repoPath) + if err != nil { + r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Failed to clone repository (attempt %d/%d): %v", i+1, maxRetries, err)) + + // Check if the directory exists and remove it before retrying + _ = os.RemoveAll(repoPath) + + // Wait before retrying + if i < maxRetries-1 { + time.Sleep(5 * time.Second) + continue + } + return "", microerror.Mask(err) + } + + // Check if the repository has content + if _, err := os.Stat(filepath.Join(repoPath, "helm")); err == nil { + break + } + + r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf("Repository not yet populated with template content (attempt %d/%d)", i+1, maxRetries)) + + // Remove the empty repository and retry + _ = os.RemoveAll(repoPath) + + if i < maxRetries-1 { + time.Sleep(5 * time.Second) + continue + } + return "", microerror.Maskf(executionFailedError, "repository not populated with template content after %d attempts", maxRetries) + } + + return repoPath, nil +} + +func (r *runner) configureSyncMethod(ctx context.Context, repoPath string) error { + switch r.flag.SyncMethod { + case "vendir": + return r.configureVendir(ctx, repoPath) + case "kustomize": + return r.configureKustomize(ctx, repoPath) + default: + return microerror.Maskf(invalidFlagError, "unsupported sync method: %s", r.flag.SyncMethod) + } +} + +func (r *runner) configureVendir(ctx context.Context, repoPath string) error { + vendirConfig := fmt.Sprintf(`apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +minimumRequiredVersion: 0.12.0 +directories: +- path: vendor + contents: + - path: %s + git: + url: %s + ref: origin/main + includePaths: + - %s/**/*`, r.flag.Name, r.flag.UpstreamRepo, r.flag.UpstreamChart) + + err := os.WriteFile(filepath.Join(repoPath, "vendir.yml"), []byte(vendirConfig), 0644) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) configureKustomize(ctx context.Context, repoPath string) error { + // TODO: Implement kustomize configuration + return nil +} + +func (r *runner) configurePatchMethod(ctx context.Context, repoPath string) error { + switch r.flag.PatchMethod { + case "script": + return r.configurePatchScript(ctx, repoPath) + case "kustomize": + return r.configurePatchKustomize(ctx, repoPath) + default: + return microerror.Maskf(invalidFlagError, "unsupported patch method: %s", r.flag.PatchMethod) + } +} + +func (r *runner) configurePatchScript(ctx context.Context, repoPath string) error { + // Create sync directory and patches subdirectory + syncDir := filepath.Join(repoPath, "sync") + patchesDir := filepath.Join(syncDir, "patches") + err := os.MkdirAll(patchesDir, 0755) + if err != nil { + return microerror.Mask(err) + } + + // Create sync.sh script + syncScript := `#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +cd "${dir}/.." + +# Stage 1 sync - intermediate to the ./vendir folder +set -x +vendir sync +helm dependency update helm/%s/ +{ set +x; } 2>/dev/null + +# Apply patches +for patch in sync/patches/*; do + if [ -f "$patch" ]; then + ./sync/patches/$(basename "$patch")/patch.sh + fi +done + +# Store diffs +rm -f ./diffs/* +for f in $(git --no-pager diff --no-exit-code --no-color --no-index vendor/%s helm --name-only) ; do + [[ "$f" == "helm/%s/Chart.yaml" ]] && continue + [[ "$f" == "helm/%s/Chart.lock" ]] && continue + [[ "$f" == "helm/%s/README.md" ]] && continue + [[ "$f" == "helm/%s/values.schema.json" ]] && continue + [[ "$f" == "helm/%s/values.yaml" ]] && continue + [[ "$f" =~ ^helm/%s/charts/.* ]] && continue + + base_file="vendor/%s/${f#"helm/"}" + [[ ! -e $base_file ]] && base_file="/dev/null" + + set +e + set -x + git --no-pager diff --no-exit-code --no-color --no-index "$base_file" "${f}" \ + > "./diffs/${f//\//__}.patch" + { set +x; } 2>/dev/null + set -e + ret=$? + if [ $ret -ne 0 ] && [ $ret -ne 1 ] ; then + exit $ret + fi +done` + + syncScript = fmt.Sprintf(syncScript, + r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, + r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name) + + err = os.WriteFile(filepath.Join(syncDir, "sync.sh"), []byte(syncScript), 0755) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) configurePatchKustomize(ctx context.Context, repoPath string) error { + // TODO: Implement kustomize patch configuration + return nil +} + +func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error { + // Rename helm chart directory + oldPath := filepath.Join(repoPath, "helm", "{APP-NAME}") + newPath := filepath.Join(repoPath, "helm", r.flag.Name) + err := os.Rename(oldPath, newPath) + if err != nil { + return microerror.Mask(err) + } + + // Replace {APP-NAME} with actual app name in all files, excluding .git directory + err = r.execCommand(ctx, repoPath, + "find", ".", "-type", "f", + "-not", "-path", "./.git/*", + "-exec", "sed", "-i", fmt.Sprintf("s/{APP-NAME}/%s/g", r.flag.Name), "{}", "+") + if err != nil { + return microerror.Mask(err) + } + + // Add team label + err = r.execCommand(ctx, repoPath, + "sed", "-i", + fmt.Sprintf(`s/app.kubernetes.io\/name: %s/app.kubernetes.io\/name: %s\n application.giantswarm.io\/team: %s/`, + r.flag.Name, r.flag.Name, r.flag.Team), + fmt.Sprintf("helm/%s/templates/_helpers.tpl", r.flag.Name)) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) setupCICD(ctx context.Context, repoPath string) error { + // Get token from custom environment variable + token := os.Getenv(r.flag.GithubToken) + if token == "" { + return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) + } + + // Run devctl repo setup with GITHUB_TOKEN set + repoFullName := fmt.Sprintf("giantswarm/%s-app", r.flag.Name) + cmd := exec.CommandContext(ctx, "devctl", "repo", "setup", repoFullName) + cmd.Dir = repoPath + + // Only show output in debug mode + if os.Getenv("LOG_LEVEL") == "debug" { + cmd.Stdout = r.stdout + cmd.Stderr = r.stderr + } + + cmd.Env = append(os.Environ(), fmt.Sprintf("GITHUB_TOKEN=%s", token)) + + err := cmd.Run() + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) commitAndPush(ctx context.Context, repoPath string) error { + // Create and checkout feature branch + branchName := "bootstrap-app" + err := r.execCommand(ctx, repoPath, "git", "checkout", "-b", branchName) + if err != nil { + return microerror.Mask(err) + } + + err = r.execCommand(ctx, repoPath, "git", "add", "-A") + if err != nil { + return microerror.Mask(err) + } + + err = r.execCommand(ctx, repoPath, "git", "commit", "-m", "Bootstrap app repository") + if err != nil { + return microerror.Mask(err) + } + + err = r.execCommand(ctx, repoPath, "git", "push", "origin", branchName) + if err != nil { + return microerror.Mask(err) + } + + // Create pull request + token := os.Getenv(r.flag.GithubToken) + if token == "" { + return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) + } + + logger := logrus.New() + logger.SetOutput(r.stdout) + + config := githubclient.Config{ + Logger: logger, + AccessToken: token, + DryRun: r.flag.DryRun, + } + + client, err := githubclient.New(config) + if err != nil { + return microerror.Mask(err) + } + + repoName := fmt.Sprintf("%s-app", r.flag.Name) + title := "Bootstrap app repository" + body := "Initial bootstrap of the app repository" + pr := &github.NewPullRequest{ + Title: github.String(title), + Head: github.String(branchName), + Base: github.String("main"), + Body: github.String(body), + MaintainerCanModify: github.Bool(true), + } + + _, err = client.CreatePullRequest(ctx, "giantswarm", repoName, pr) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) execCommand(ctx context.Context, dir string, command string, args ...string) error { + cmd := exec.CommandContext(ctx, command, args...) + if dir != "" { + cmd.Dir = dir + } + + // Only show command output in debug mode + if os.Getenv("LOG_LEVEL") == "debug" { + cmd.Stdout = r.stdout + cmd.Stderr = r.stderr + } + + return cmd.Run() +} diff --git a/cmd/app/command.go b/cmd/app/command.go new file mode 100644 index 000000000..bbf943d91 --- /dev/null +++ b/cmd/app/command.go @@ -0,0 +1,72 @@ +package app + +import ( + "io" + "os" + + "github.com/giantswarm/microerror" + "github.com/giantswarm/micrologger" + "github.com/spf13/cobra" + + "github.com/giantswarm/devctl/v7/cmd/app/bootstrap" +) + +const ( + name = "app" + description = "Work with Giant Swarm app repositories." +) + +type Config struct { + Logger micrologger.Logger + Stderr io.Writer + Stdout io.Writer +} + +func New(config Config) (*cobra.Command, error) { + if config.Logger == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config) + } + if config.Stderr == nil { + config.Stderr = os.Stderr + } + if config.Stdout == nil { + config.Stdout = os.Stdout + } + + var err error + + var bootstrapCmd *cobra.Command + { + c := bootstrap.Config{ + Logger: config.Logger, + Stderr: config.Stderr, + Stdout: config.Stdout, + } + + bootstrapCmd, err = bootstrap.New(c) + if err != nil { + return nil, microerror.Mask(err) + } + } + + f := &flag{} + + r := &runner{ + flag: f, + logger: config.Logger, + stderr: config.Stderr, + stdout: config.Stdout, + } + + c := &cobra.Command{ + Use: name, + Short: description, + Long: description, + RunE: r.Run, + } + + f.Init(c) + c.AddCommand(bootstrapCmd) + + return c, nil +} diff --git a/cmd/app/error.go b/cmd/app/error.go new file mode 100644 index 000000000..7d56c7efe --- /dev/null +++ b/cmd/app/error.go @@ -0,0 +1,21 @@ +package app + +import "github.com/giantswarm/microerror" + +var invalidConfigError = µerror.Error{ + Kind: "invalidConfigError", +} + +// IsInvalidConfig asserts invalidConfigError. +func IsInvalidConfig(err error) bool { + return microerror.Cause(err) == invalidConfigError +} + +var invalidFlagError = µerror.Error{ + Kind: "invalidFlagError", +} + +// IsInvalidFlag asserts invalidFlagError. +func IsInvalidFlag(err error) bool { + return microerror.Cause(err) == invalidFlagError +} diff --git a/cmd/app/flag.go b/cmd/app/flag.go new file mode 100644 index 000000000..5c54b16f5 --- /dev/null +++ b/cmd/app/flag.go @@ -0,0 +1,16 @@ +package app + +import ( + "github.com/spf13/cobra" +) + +type flag struct{} + +func (f *flag) Init(cmd *cobra.Command) { + // No flags for the app command itself + // Subcommands will have their own flags +} + +func (f *flag) Validate() error { + return nil +} diff --git a/cmd/app/runner.go b/cmd/app/runner.go new file mode 100644 index 000000000..3f5612e01 --- /dev/null +++ b/cmd/app/runner.go @@ -0,0 +1,42 @@ +package app + +import ( + "context" + "io" + + "github.com/giantswarm/microerror" + "github.com/giantswarm/micrologger" + "github.com/spf13/cobra" +) + +type runner struct { + flag *flag + logger micrologger.Logger + stdout io.Writer + stderr io.Writer +} + +func (r *runner) Run(cmd *cobra.Command, args []string) error { + ctx := context.Background() + + err := r.flag.Validate() + if err != nil { + return microerror.Mask(err) + } + + err = r.run(ctx, cmd, args) + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) error { + err := cmd.Help() + if err != nil { + return microerror.Mask(err) + } + + return nil +} diff --git a/cmd/command.go b/cmd/command.go index dc38fce64..0a32b5d0a 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -9,6 +9,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/giantswarm/devctl/v7/cmd/app" "github.com/giantswarm/devctl/v7/cmd/completion" "github.com/giantswarm/devctl/v7/cmd/gen" "github.com/giantswarm/devctl/v7/cmd/release" @@ -39,6 +40,20 @@ func New(config Config) (*cobra.Command, error) { var err error + var appCmd *cobra.Command + { + c := app.Config{ + Logger: config.Logger, + Stderr: config.Stderr, + Stdout: config.Stdout, + } + + appCmd, err = app.New(c) + if err != nil { + return nil, microerror.Mask(err) + } + } + var completionCmd *cobra.Command { c := completion.Config{ @@ -144,6 +159,7 @@ func New(config Config) (*cobra.Command, error) { f.Init(c) + c.AddCommand(appCmd) c.AddCommand(completionCmd) c.AddCommand(genCmd) c.AddCommand(releaseCmd) diff --git a/go.mod b/go.mod index f534f193a..43dda44a4 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( ) require ( + github.com/briandowns/spinner v1.23.2 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/giantswarm/k8smetadata v0.24.0 // indirect github.com/go-kit/log v0.2.1 // indirect @@ -54,6 +55,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 640302e81..99e787fb8 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdn github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/briandowns/spinner v1.23.2 h1:Zc6ecUnI+YzLmJniCfDNaMbW0Wid1d5+qcTq4L2FW8w= +github.com/briandowns/spinner v1.23.2/go.mod h1:LaZeM4wm2Ywy6vO571mvhQNRcWfRUnXOs0RcKV0wYKM= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -193,6 +195,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/githubclient/client_repository.go b/pkg/githubclient/client_repository.go index ef773da18..1a5b9ee69 100644 --- a/pkg/githubclient/client_repository.go +++ b/pkg/githubclient/client_repository.go @@ -441,3 +441,47 @@ func (c *Client) SetRepositoryWebhooks(ctx context.Context, repository *github.R } return nil } + +func (c *Client) CreateFromTemplate(ctx context.Context, templateOwner, templateRepo, newOwner string, repository *github.Repository) (*github.Repository, error) { + c.logger.Infof("creating repository %s/%s from template %s/%s", newOwner, repository.GetName(), templateOwner, templateRepo) + + underlyingClient := c.getUnderlyingClient(ctx) + + req := &github.TemplateRepoRequest{ + Name: repository.Name, + Owner: github.String(newOwner), + Description: repository.Description, + Private: repository.Private, + } + + repo, _, err := underlyingClient.Repositories.CreateFromTemplate(ctx, templateOwner, templateRepo, req) + if err != nil { + if c.dryRun { + c.logger.Infof("[dry-run] would have created repository %s/%s from template %s/%s", newOwner, repository.GetName(), templateOwner, templateRepo) + return repository, nil + } + return nil, microerror.Mask(err) + } + + c.logger.Infof("created repository %s/%s from template %s/%s", newOwner, repository.GetName(), templateOwner, templateRepo) + + return repo, nil +} + +func (c *Client) CreatePullRequest(ctx context.Context, owner string, repo string, pr *github.NewPullRequest) (*github.PullRequest, error) { + c.logger.Infof("creating pull request in repository %s/%s", owner, repo) + + if c.dryRun { + c.logger.Infof("would create pull request: %s from %s to %s", *pr.Title, *pr.Head, *pr.Base) + return nil, nil + } + + underlyingClient := c.getUnderlyingClient(ctx) + pullRequest, _, err := underlyingClient.PullRequests.Create(ctx, owner, repo, pr) + if err != nil { + return nil, microerror.Mask(err) + } + + c.logger.Infof("created pull request #%d: %s", pullRequest.GetNumber(), pullRequest.GetHTMLURL()) + return pullRequest, nil +} From f843c01fbb36a4ee79e6ff7894a4b5345774d39d Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Thu, 27 Mar 2025 14:24:41 +0100 Subject: [PATCH 02/11] Refactor app bootstrap process: streamline repository creation, placeholder replacement, and CI/CD setup --- cmd/app/bootstrap/runner.go | 126 +++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 52 deletions(-) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index de0868a06..da05bed25 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -46,7 +46,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s := spinner.New(spinner.CharSets[14], 100*time.Millisecond) s.Writer = r.stdout - // 1. Create repository from app-template + // Create repository from app-template s.Suffix = " Creating repository from template..." s.Start() err := r.createRepository(ctx, r.flag.Name, "giantswarm", "template-app") @@ -63,7 +63,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err time.Sleep(10 * time.Second) s.Stop() - // 2. Clone repository locally + // Clone repository locally s.Suffix = " Cloning repository..." s.Start() repoPath, err := r.cloneRepository(ctx) @@ -74,7 +74,18 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s.Stop() fmt.Fprintln(r.stdout, "✓ Repository cloned locally") - // 3. Configure sync method (vendir/kustomize) + // Replace placeholders + s.Suffix = " Replacing placeholders..." + s.Start() + err = r.replacePlaceholders(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Placeholders replaced") + + // Configure sync method (vendir/kustomize) s.Suffix = fmt.Sprintf(" Configuring sync method (%s)...", r.flag.SyncMethod) s.Start() err = r.configureSyncMethod(ctx, repoPath) @@ -85,7 +96,7 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s.Stop() fmt.Fprintln(r.stdout, "✓ Sync method configured") - // 4. Configure patch method (script/kustomize) + // Configure patch method (script/kustomize) s.Suffix = fmt.Sprintf(" Configuring patch method (%s)...", r.flag.PatchMethod) s.Start() err = r.configurePatchMethod(ctx, repoPath) @@ -96,38 +107,38 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s.Stop() fmt.Fprintln(r.stdout, "✓ Patch method configured") - // 5. Replace placeholders - s.Suffix = " Replacing placeholders..." + // Setup CI/CD without branch protection + s.Suffix = " Setting up CI/CD..." s.Start() - err = r.replacePlaceholders(ctx, repoPath) + err = r.setupCICD(ctx, repoPath) if err != nil { s.Stop() return microerror.Mask(err) } s.Stop() - fmt.Fprintln(r.stdout, "✓ Placeholders replaced") + fmt.Fprintln(r.stdout, "✓ CI/CD setup complete") - // 6. Setup CI/CD - s.Suffix = " Setting up CI/CD..." + // Initial commit and push + s.Suffix = " Pushing changes to main branch..." s.Start() - err = r.setupCICD(ctx, repoPath) + err = r.commitAndPush(ctx, repoPath) if err != nil { s.Stop() return microerror.Mask(err) } s.Stop() - fmt.Fprintln(r.stdout, "✓ CI/CD setup complete") + fmt.Fprintln(r.stdout, "✓ Changes pushed to main branch") - // 7. Initial commit and push - s.Suffix = " Creating pull request..." + // Enable branch protection + s.Suffix = " Enabling branch protection..." s.Start() - err = r.commitAndPush(ctx, repoPath) + err = r.enableBranchProtection(ctx, repoPath) if err != nil { s.Stop() return microerror.Mask(err) } s.Stop() - fmt.Fprintln(r.stdout, "✓ Pull request created") + fmt.Fprintln(r.stdout, "✓ Branch protection enabled") fmt.Fprintf(r.stdout, "\n✨ Successfully bootstrapped app repository %s\n", r.flag.Name) @@ -232,24 +243,41 @@ func (r *runner) configureSyncMethod(ctx context.Context, repoPath string) error } func (r *runner) configureVendir(ctx context.Context, repoPath string) error { + // Create vendir.yml - main needs to be replaced with the latest upstream release (eg &version "v1.17.2") vendirConfig := fmt.Sprintf(`apiVersion: vendir.k14s.io/v1alpha1 kind: Config minimumRequiredVersion: 0.12.0 directories: - path: vendor contents: - - path: %s + - path: . git: url: %s - ref: origin/main + ref: main includePaths: - - %s/**/*`, r.flag.Name, r.flag.UpstreamRepo, r.flag.UpstreamChart) + - %s/**/* +- path: helm/%s/templates + contents: + - path: . + directory: + path: vendor/%s/templates +`, + r.flag.UpstreamRepo, + r.flag.UpstreamChart, + r.flag.Name, + r.flag.UpstreamChart) err := os.WriteFile(filepath.Join(repoPath, "vendir.yml"), []byte(vendirConfig), 0644) if err != nil { return microerror.Mask(err) } + // Run initial sync + err = r.execCommand(ctx, repoPath, "vendir", "sync") + if err != nil { + return microerror.Mask(err) + } + return nil } @@ -361,6 +389,15 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error return microerror.Mask(err) } + // Replace team in CODEOWNERS + err = r.execCommand(ctx, repoPath, + "sed", "-i", + fmt.Sprintf("s/@giantswarm\\/team-honeybadger/@giantswarm\\/team-%s/g", r.flag.Team), + "CODEOWNERS") + if err != nil { + return microerror.Mask(err) + } + // Add team label err = r.execCommand(ctx, repoPath, "sed", "-i", @@ -383,7 +420,7 @@ func (r *runner) setupCICD(ctx context.Context, repoPath string) error { // Run devctl repo setup with GITHUB_TOKEN set repoFullName := fmt.Sprintf("giantswarm/%s-app", r.flag.Name) - cmd := exec.CommandContext(ctx, "devctl", "repo", "setup", repoFullName) + cmd := exec.CommandContext(ctx, "devctl", "repo", "setup", repoFullName, "--disable-branch-protection") cmd.Dir = repoPath // Only show output in debug mode @@ -403,14 +440,7 @@ func (r *runner) setupCICD(ctx context.Context, repoPath string) error { } func (r *runner) commitAndPush(ctx context.Context, repoPath string) error { - // Create and checkout feature branch - branchName := "bootstrap-app" - err := r.execCommand(ctx, repoPath, "git", "checkout", "-b", branchName) - if err != nil { - return microerror.Mask(err) - } - - err = r.execCommand(ctx, repoPath, "git", "add", "-A") + err := r.execCommand(ctx, repoPath, "git", "add", "-A") if err != nil { return microerror.Mask(err) } @@ -420,43 +450,35 @@ func (r *runner) commitAndPush(ctx context.Context, repoPath string) error { return microerror.Mask(err) } - err = r.execCommand(ctx, repoPath, "git", "push", "origin", branchName) + err = r.execCommand(ctx, repoPath, "git", "push", "origin", "main") if err != nil { return microerror.Mask(err) } - // Create pull request + return nil +} + +func (r *runner) enableBranchProtection(ctx context.Context, repoPath string) error { + // Get token from custom environment variable token := os.Getenv(r.flag.GithubToken) if token == "" { return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) } - logger := logrus.New() - logger.SetOutput(r.stdout) - - config := githubclient.Config{ - Logger: logger, - AccessToken: token, - DryRun: r.flag.DryRun, - } + // Run devctl repo setup with GITHUB_TOKEN set (without --disable-branch-protection) + repoFullName := fmt.Sprintf("giantswarm/%s-app", r.flag.Name) + cmd := exec.CommandContext(ctx, "devctl", "repo", "setup", repoFullName) + cmd.Dir = repoPath - client, err := githubclient.New(config) - if err != nil { - return microerror.Mask(err) + // Only show output in debug mode + if os.Getenv("LOG_LEVEL") == "debug" { + cmd.Stdout = r.stdout + cmd.Stderr = r.stderr } - repoName := fmt.Sprintf("%s-app", r.flag.Name) - title := "Bootstrap app repository" - body := "Initial bootstrap of the app repository" - pr := &github.NewPullRequest{ - Title: github.String(title), - Head: github.String(branchName), - Base: github.String("main"), - Body: github.String(body), - MaintainerCanModify: github.Bool(true), - } + cmd.Env = append(os.Environ(), fmt.Sprintf("GITHUB_TOKEN=%s", token)) - _, err = client.CreatePullRequest(ctx, "giantswarm", repoName, pr) + err := cmd.Run() if err != nil { return microerror.Mask(err) } From f7c9d9adb5166aa43758ffc0d9d55975c99c2e09 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Thu, 27 Mar 2025 14:54:58 +0100 Subject: [PATCH 03/11] Add workflow and Makefile generation, and create PR for giantswarm/github --- cmd/app/bootstrap/runner.go | 180 ++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index da05bed25..c58c8099f 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/cobra" "github.com/giantswarm/devctl/v7/pkg/githubclient" + "gopkg.in/yaml.v3" ) type runner struct { @@ -118,6 +119,28 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s.Stop() fmt.Fprintln(r.stdout, "✓ CI/CD setup complete") + // Generate workflows and Makefile + s.Suffix = " Generating workflows and Makefile..." + s.Start() + err = r.generateWorkflowsAndMakefile(ctx, repoPath) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ Workflows and Makefile generated") + + // Create PR for giantswarm/github repository + s.Suffix = " Creating PR for giantswarm/github..." + s.Start() + err = r.createGithubRepoPR(ctx) + if err != nil { + s.Stop() + return microerror.Mask(err) + } + s.Stop() + fmt.Fprintln(r.stdout, "✓ PR created in giantswarm/github") + // Initial commit and push s.Suffix = " Pushing changes to main branch..." s.Start() @@ -500,3 +523,160 @@ func (r *runner) execCommand(ctx context.Context, dir string, command string, ar return cmd.Run() } + +func (r *runner) generateWorkflowsAndMakefile(ctx context.Context, repoPath string) error { + // Generate workflows + err := r.execCommand(ctx, repoPath, "devctl", "gen", "workflows", + "--flavour", "app", + "--install-update-chart") + if err != nil { + return microerror.Mask(err) + } + + // Generate Makefile + err = r.execCommand(ctx, repoPath, "devctl", "gen", "Makefile", + "--flavour", "app", + "--language", "generic") + if err != nil { + return microerror.Mask(err) + } + + return nil +} + +func (r *runner) createGithubRepoPR(ctx context.Context) error { + token := os.Getenv(r.flag.GithubToken) + if token == "" { + return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) + } + + // Create a logger that only outputs in debug mode + logger := logrus.New() + if os.Getenv("LOG_LEVEL") == "debug" { + logger.SetOutput(r.stdout) + } else { + logger.SetOutput(io.Discard) + } + + // Setup GitHub client + config := githubclient.Config{ + Logger: logger, + AccessToken: token, + DryRun: r.flag.DryRun, + } + + client, err := githubclient.New(config) + if err != nil { + return microerror.Mask(err) + } + + // Clone giantswarm/github repository + repoPath := filepath.Join(os.TempDir(), "github") + _ = os.RemoveAll(repoPath) // Clean up any existing directory + + err = r.execCommand(ctx, "", "git", "clone", "git@github.com:giantswarm/github.git", repoPath) + if err != nil { + return microerror.Mask(err) + } + + // Create new branch + branchName := fmt.Sprintf("add-%s-app", r.flag.Name) + err = r.execCommand(ctx, repoPath, "git", "checkout", "-b", branchName) + if err != nil { + return microerror.Mask(err) + } + + // Update team YAML file + teamFile := filepath.Join(repoPath, "repositories", fmt.Sprintf("team-%s.yaml", r.flag.Team)) + entry := fmt.Sprintf(`- name: %s-app + componentType: service + gen: + flavours: + - app + language: generic + installUpdateChart: true +`, r.flag.Name) + + // Read existing file + content, err := os.ReadFile(teamFile) + if err != nil { + return microerror.Mask(err) + } + + // Parse YAML + var data map[string]interface{} + err = yaml.Unmarshal(content, &data) + if err != nil { + return microerror.Mask(err) + } + + // Add new entry in alphabetical order + repositories := data["repositories"].([]interface{}) + var newEntry map[string]interface{} + err = yaml.Unmarshal([]byte(entry), &newEntry) + if err != nil { + return microerror.Mask(err) + } + + // Insert entry in alphabetical order + inserted := false + for i, repo := range repositories { + repoMap := repo.(map[string]interface{}) + if repoMap["name"].(string) > fmt.Sprintf("%s-app", r.flag.Name) { + repositories = append(repositories[:i], append([]interface{}{newEntry}, repositories[i:]...)...) + inserted = true + break + } + } + if !inserted { + repositories = append(repositories, newEntry) + } + data["repositories"] = repositories + + // Write updated YAML + updatedContent, err := yaml.Marshal(data) + if err != nil { + return microerror.Mask(err) + } + + err = os.WriteFile(teamFile, updatedContent, 0644) + if err != nil { + return microerror.Mask(err) + } + + // Commit changes + err = r.execCommand(ctx, repoPath, "git", "add", teamFile) + if err != nil { + return microerror.Mask(err) + } + + err = r.execCommand(ctx, repoPath, "git", "commit", "-m", fmt.Sprintf("Add %s-app to team-%s repositories", r.flag.Name, r.flag.Team)) + if err != nil { + return microerror.Mask(err) + } + + // Push changes using token + err = r.execCommand(ctx, repoPath, "git", "push", "origin", branchName) + if err != nil { + return microerror.Mask(err) + } + + // Create pull request using githubclient + prTitle := fmt.Sprintf("Add %s-app to team-%s repositories", r.flag.Name, r.flag.Team) + prBody := fmt.Sprintf("This PR adds the %s-app to the team-%s repositories configuration.", r.flag.Name, r.flag.Team) + + pr := &github.NewPullRequest{ + Title: github.String(prTitle), + Head: github.String(branchName), + Base: github.String("main"), + Body: github.String(prBody), + MaintainerCanModify: github.Bool(true), + } + + _, err = client.CreatePullRequest(ctx, "giantswarm", "github", pr) + if err != nil { + return microerror.Mask(err) + } + + return nil +} From 44ac69307c3b4369dd8abce0a92220edfc8c824e Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Thu, 27 Mar 2025 16:00:39 +0100 Subject: [PATCH 04/11] Enhance GitHub repository PR creation: return PR URL and update success message with next steps --- cmd/app/bootstrap/runner.go | 92 +++++++++++++++++++++++-------------- go.mod | 3 +- 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index c58c8099f..43a4be6b0 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -1,6 +1,7 @@ package bootstrap import ( + "bytes" "context" "fmt" "io" @@ -133,7 +134,8 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err // Create PR for giantswarm/github repository s.Suffix = " Creating PR for giantswarm/github..." s.Start() - err = r.createGithubRepoPR(ctx) + var prURL string + err, prURL = r.createGithubRepoPR(ctx) if err != nil { s.Stop() return microerror.Mask(err) @@ -163,7 +165,18 @@ func (r *runner) run(ctx context.Context, cmd *cobra.Command, args []string) err s.Stop() fmt.Fprintln(r.stdout, "✓ Branch protection enabled") - fmt.Fprintf(r.stdout, "\n✨ Successfully bootstrapped app repository %s\n", r.flag.Name) + fmt.Fprintf(r.stdout, "\n✨ Successfully bootstrapped app repository %s\n\n", r.flag.Name) + fmt.Fprintf(r.stdout, "Next steps:\n") + fmt.Fprintf(r.stdout, "1. Visit your new repository: https://github.com/giantswarm/%s-app\n", r.flag.Name) + if prURL != "" { + fmt.Fprintf(r.stdout, "2. Review and merge the PR: %s\n", prURL) + } else { + fmt.Fprintf(r.stdout, "2. Review and merge the PR: https://github.com/giantswarm/github/pulls\n") + } + fmt.Fprintf(r.stdout, "3. Update the Chart.yaml with appropriate metadata and version\n") + fmt.Fprintf(r.stdout, "4. Configure your image registry in values.yaml\n") + fmt.Fprintf(r.stdout, "5. Create a release by pushing a tag (e.g., v0.1.0)\n") + fmt.Fprintf(r.stdout, "\nFor more information, visit: https://intranet.giantswarm.io/docs/dev-and-releng/app-developer-guide/\n") return nil } @@ -534,7 +547,7 @@ func (r *runner) generateWorkflowsAndMakefile(ctx context.Context, repoPath stri } // Generate Makefile - err = r.execCommand(ctx, repoPath, "devctl", "gen", "Makefile", + err = r.execCommand(ctx, repoPath, "devctl", "gen", "makefile", "--flavour", "app", "--language", "generic") if err != nil { @@ -544,10 +557,10 @@ func (r *runner) generateWorkflowsAndMakefile(ctx context.Context, repoPath stri return nil } -func (r *runner) createGithubRepoPR(ctx context.Context) error { +func (r *runner) createGithubRepoPR(ctx context.Context) (error, string) { token := os.Getenv(r.flag.GithubToken) if token == "" { - return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken) + return microerror.Maskf(envVarNotFoundError, "environment variable %#q not found", r.flag.GithubToken), "" } // Create a logger that only outputs in debug mode @@ -567,7 +580,7 @@ func (r *runner) createGithubRepoPR(ctx context.Context) error { client, err := githubclient.New(config) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Clone giantswarm/github repository @@ -576,19 +589,19 @@ func (r *runner) createGithubRepoPR(ctx context.Context) error { err = r.execCommand(ctx, "", "git", "clone", "git@github.com:giantswarm/github.git", repoPath) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Create new branch branchName := fmt.Sprintf("add-%s-app", r.flag.Name) err = r.execCommand(ctx, repoPath, "git", "checkout", "-b", branchName) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Update team YAML file teamFile := filepath.Join(repoPath, "repositories", fmt.Sprintf("team-%s.yaml", r.flag.Team)) - entry := fmt.Sprintf(`- name: %s-app + newEntry := fmt.Sprintf(`- name: %s-app componentType: service gen: flavours: @@ -600,65 +613,70 @@ func (r *runner) createGithubRepoPR(ctx context.Context) error { // Read existing file content, err := os.ReadFile(teamFile) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } - // Parse YAML - var data map[string]interface{} - err = yaml.Unmarshal(content, &data) + // Parse YAML as a list, preserving comments + decoder := yaml.NewDecoder(bytes.NewReader(content)) + decoder.KnownFields(true) + + var repositories []map[string]interface{} + err = decoder.Decode(&repositories) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } - // Add new entry in alphabetical order - repositories := data["repositories"].([]interface{}) - var newEntry map[string]interface{} - err = yaml.Unmarshal([]byte(entry), &newEntry) + // Parse new entry + var newEntries []map[string]interface{} + err = yaml.Unmarshal([]byte(newEntry), &newEntries) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } + newRepo := newEntries[0] // Insert entry in alphabetical order inserted := false for i, repo := range repositories { - repoMap := repo.(map[string]interface{}) - if repoMap["name"].(string) > fmt.Sprintf("%s-app", r.flag.Name) { - repositories = append(repositories[:i], append([]interface{}{newEntry}, repositories[i:]...)...) + if repo["name"].(string) > fmt.Sprintf("%s-app", r.flag.Name) { + repositories = append(repositories[:i], append([]map[string]interface{}{newRepo}, repositories[i:]...)...) inserted = true break } } if !inserted { - repositories = append(repositories, newEntry) + repositories = append(repositories, newRepo) } - data["repositories"] = repositories - // Write updated YAML - updatedContent, err := yaml.Marshal(data) + // Add YAML header comment and marshal with proper indentation + var buf bytes.Buffer + buf.WriteString("# yaml-language-server: $schema=../.github/repositories.schema.json\n") + encoder := yaml.NewEncoder(&buf) + encoder.SetIndent(2) + err = encoder.Encode(repositories) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } - err = os.WriteFile(teamFile, updatedContent, 0644) + err = os.WriteFile(teamFile, buf.Bytes(), 0644) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Commit changes err = r.execCommand(ctx, repoPath, "git", "add", teamFile) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } err = r.execCommand(ctx, repoPath, "git", "commit", "-m", fmt.Sprintf("Add %s-app to team-%s repositories", r.flag.Name, r.flag.Team)) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Push changes using token err = r.execCommand(ctx, repoPath, "git", "push", "origin", branchName) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } // Create pull request using githubclient @@ -673,10 +691,14 @@ func (r *runner) createGithubRepoPR(ctx context.Context) error { MaintainerCanModify: github.Bool(true), } - _, err = client.CreatePullRequest(ctx, "giantswarm", "github", pr) + createdPR, err := client.CreatePullRequest(ctx, "giantswarm", "github", pr) if err != nil { - return microerror.Mask(err) + return microerror.Mask(err), "" } - return nil + // Store PR URL for final message + if createdPR.HTMLURL != nil { + return nil, *createdPR.HTMLURL + } + return nil, "" } diff --git a/go.mod b/go.mod index 43dda44a4..441f2be75 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go v1.55.6 github.com/blang/semver v3.5.1+incompatible github.com/bmatcuk/doublestar/v4 v4.8.1 + github.com/briandowns/spinner v1.23.2 github.com/buger/goterm v1.0.4 github.com/fatih/color v1.18.0 github.com/giantswarm/microerror v0.4.1 @@ -24,12 +25,12 @@ require ( github.com/spf13/pflag v1.0.6 golang.org/x/net v0.37.0 golang.org/x/oauth2 v0.28.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.32.3 sigs.k8s.io/yaml v1.4.0 ) require ( - github.com/briandowns/spinner v1.23.2 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/giantswarm/k8smetadata v0.24.0 // indirect github.com/go-kit/log v0.2.1 // indirect From 0e18828ab4bcbe3cf4a7972580bad3afd71af4f8 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Fri, 28 Mar 2025 15:05:26 +0100 Subject: [PATCH 05/11] Update README and enhance placeholder replacement in bootstrap process --- README.md | 18 +++++++++++++++++- cmd/app/bootstrap/runner.go | 26 ++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 508b9a096..6b5f9e066 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,24 @@ ## Installation +> **Important**: We recommend downloading the latest release from our [releases page](https://github.com/giantswarm/devctl/releases) rather than using `go install`. This ensures you get a properly built binary with: +> - Correct version information +> - Git commit information for traceability +> - Build timestamps +> - All necessary build flags +> - Generated code and mocks for testing +> +> While `go install` will work, it won't include this important metadata and may miss generated code that helps with debugging and version tracking. + ```bash +# Not recommended go install github.com/giantswarm/devctl/v7@latest ``` +**Recommended**: Download the latest release from + +https://github.com/giantswarm/devctl/releases + ## Features ### App Management (`devctl app`) @@ -124,10 +138,12 @@ devctl completion fish > ~/.config/fish/completions/devctl.fish ### Building from Source +If you want to build from source, use the Makefile which ensures all necessary steps are executed: + ```bash git clone https://github.com/giantswarm/devctl.git cd devctl -go build +make build # Build with proper flags and metadata ``` ### Running Tests diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index 43a4be6b0..58ce98336 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -16,9 +16,9 @@ import ( "github.com/google/go-github/v70/github" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "gopkg.in/yaml.v3" "github.com/giantswarm/devctl/v7/pkg/githubclient" - "gopkg.in/yaml.v3" ) type runner struct { @@ -416,7 +416,29 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error return microerror.Mask(err) } - // Replace {APP-NAME} with actual app name in all files, excluding .git directory + // First replace GitHub URLs that need the -app suffix + err = r.execCommand(ctx, repoPath, + "find", ".", "-type", "f", + "-not", "-path", "./.git/*", + "-exec", "sed", "-i", + fmt.Sprintf("s|github.com/giantswarm/{APP-NAME}|github.com/giantswarm/%s-app|g", r.flag.Name), + "{}", "+") + if err != nil { + return microerror.Mask(err) + } + + // Then replace CircleCI URLs that need the -app suffix + err = r.execCommand(ctx, repoPath, + "find", ".", "-type", "f", + "-not", "-path", "./.git/*", + "-exec", "sed", "-i", + fmt.Sprintf("s|gh/giantswarm/{APP-NAME}/|gh/giantswarm/%s-app/|g", r.flag.Name), + "{}", "+") + if err != nil { + return microerror.Mask(err) + } + + // Then do the general replacement for all other cases err = r.execCommand(ctx, repoPath, "find", ".", "-type", "f", "-not", "-path", "./.git/*", From c107657baa2193aa5a0d1c225584caa7f7008bb1 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Fri, 28 Mar 2025 15:07:19 +0100 Subject: [PATCH 06/11] removed logs --- info.md | 345 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 345 insertions(+) create mode 100644 info.md diff --git a/info.md b/info.md new file mode 100644 index 000000000..469393aac --- /dev/null +++ b/info.md @@ -0,0 +1,345 @@ +This is a transcript of a recording about how to install an app in a Giant Swarm Workloadcluster. The app already has a helm chart. Please analyze the transcript. + +# Transcript   +Okay, I didn't remember, but Kabbage starts using Vendir for all the… Vendir. Vendir, yeah, which we use it for the last hackathon, the Envoy Gateway, and I even like it more than the sub-module thing, the Git sub-module stuff, because, yeah, it's straightforward. So I found this one. Yes, yes, this is basically where you have to kind of say in which directory you want to include the app, right, and you can also put some constraints, like versions or whatever. If you go a bit down, you will see the example config, which looks like this config, and then you put there where it has to be the path to be updated and everything. So how they do, right, is they put the upstream chart as a dependency, and then you use it within your chart, just the dependency, and you can add, for example, in the helper the labels if you want, something like that. That's one way to do it, probably, it's better than the sub-module part. So how do I need to start? Do I need to create the transform app from the template? Yeah, yeah, you need to create first the… From the template. From the template, the repo, you can call it n8n-app, a high phone app. Yeah. Use this template here? Yeah. In our repository? Yeah. Cool, yeah. So… And then… Transform. Okay, I'm going to say if it's public or not. No, this can be public. And then, yeah. I don't think Renovate… Yeah, well, Renovate can give you, I think, updates on the charts from upstream, so it can help you, yeah. How do we describe that? Describe that? M-chart. I'm going to do M-chart. Great repository. Good. Okay. So, you can clone it. So, I can clone it locally, yeah. Okay. And… Yeah, I've cloned it. So, going back to Rendier. So, the upstream repository, I still fork it? No, you have to just put in the config. You copy, for example, the… But here, it says upstream. The upstream repository is a fork of the original repository. It holds the charts. They use the… They use the… I think this is, if you want to contribute back, to have it always a base repo. I don't think here it's necessary, because what Rendier does is, from the URL you give it, it pulls all the information, and then you just add the chart or whatever you want to update on the right path that you have put, but nothing else. So, I don't think you have to fork it, at least not right now. Okay. So, this… I need something like this here? Yeah, you need something like that. And that's a Rendier YAML inside of the repository? Let me see. Yeah, Rendier YAML, I think it is. Yeah, YAML. Oh, YAML. It's not YAML. All right. And then the vendor is… Hmm. What is that path? So, the first URL is the charts, and the included path is what you want to include from that. So, in your case, it's charts and 8n, right? If you go to the upstream. The upstream repo is here. It was… It was… mcharts, charts and 8n. No. Huh? No, no. Charts, yeah. Charts and then 8n. Yeah. Okay, yeah. Then you have to specify that path on the community chart, mchart. But… But I… Let me share this one. So, here I put the repository, and here… You can put vendor. Vendor. Vendor. Sorry, vendor. You have to put vendor. You have to put vendor in the path. No. Vendor is here. Okay, yeah. The path is… N8n? Where it's going to land. It doesn't really matter, but choose whatever you want. Because it's… You can put N8n, for example. Yeah, yeah, yeah. That makes sense. All right. Okay. Because then in the next path is where you are going to say, okay, from my local vendor N8 folder, copy everything to my helm chart template, for example. And it will update all the templates. Okay. Okay. And the second one, I don't need. Dependents, charts, dependencies. Or let me check if there is anything like that. No. All right. I only know that there are some dependencies regarding Bitnami. But I guess that's linked. But you need them. You need them. Hmm? You need… But they are linked. Yeah, but you are not copying the chart.yaml, right? Or you are. I mean, if you are copying the chart.yaml and overwriting the ask, the problem is that our chart.yaml is… It has a specific format, for example, with the log and everything. So what is doing, for example, Gavit here is copying also the charts dependency, which… Yeah, that could work. Yeah. That works. Yeah. So… But it's a different repository. It's a different repository. Yeah, but it is put in the charts folder. If you go one level up in GitHub. Here? No, it's not. Yeah. You don't have charts. Yeah. You have charts folder, right? And there you have… Ah, but it's not pull. No, no. It's… I mean, the dependency is totally external. Yeah, yeah. But sometimes they build them or push them with the dependency. No, I mean, the repository is another one, right? So if you give that to Helm, it would not pull from the same repository. It would pull from another repository. Hmm. Let me think about that, how it can be done. I guess just copying this from the chart demo, right? There's no other way around. So I copy that into our chart demo? Yeah. I mean, this is the easiest one, yeah. And I don't think dependencies change so often, right? I mean… It's… Yeah. Some maintenance might be necessary because… I mean, Vendyr is not taking care of changing the versions of the dependencies, right? For example, yeah. Yeah. It could be that Vendyr can update. No, it's not. No. I know that BNTech made a script for such cases, but using the submodels. It was trying to parse for updates and other things, but… It's not using this approach. Yeah. Okay. I think I have the Vendyr part ready, then… Then you have to do what is… Well, first, if you have just forked the app template, you have to replace the… The HEM? The HEM app variable, because otherwise it will fail when you build it. So… The template app in the redmi tells you what has to be done. Okay. I think it's just… Where is it? Ah, no. It's the intranet. It's the intranet page. It's this one. How to add? It's in the redmi, but you have to follow it. You can put it in the repo. Okay. First, I need to go to GitHub. GitHub? Yeah, you can use this. I'm giving you… Because we might need to update it in the template, too. But you can run DeepCTL to set up the repo for you instead of… Of going to… Doing it manually. Okay. So the first create and configure your repo can be replaced by DeepCTL. I will try to update this today. No, I don't know how old my DeepCTL is. Is there… Is there an update command for DeepCTL? DeepCTL has an update command. DeepCTL has an update command. Maybe my DeepCTL is too old to have the… Ah, it could be. …loading DeepCTL… …and build it. Okay. Yeah, you can do it manually if you want. It's just setting the right component. But I don't know if it's super up to date, this page, to be honest. I know that DeepCTL does for you. Because, also, you have setting, change it. And now the protection rules works a bit different. But, yeah, it should be similar. Yeah, I'm installing the binary. Yeah. Jumped from 5.15 or something like that to 7.1.2 DeepCTL. So what's the command, DeepCTL? DeepCTL repo setup and the URL. DeepCTL repo setup. DeepCTL repo setup. And then I need repo setup. How does the repository look like? It's without ATTPS. Like, that's the Antron. Only Giant Swarm, N8N app. Yeah. N variable not found. GitHub token was not found. Okay, yeah. You have to put the GitHub token variable. Otherwise, you cannot do it. Okay. I should have the token somewhere. I'm just not sure where, because I cannot find it in my environment. Let me check. I think it's... Do you have tokens generated? Yeah, let me see. I have... GitHub OPCTL token. Yeah, now it works. All right. Completed. Good. Then you can move forward. The resources, because we are using Bendy, is what is different. You can move to 3, which is in SuiteQuality. You have to check the chart.yml. Yeah. But if you just copy from the app, I think it's fine. But how to use AVS? By default, in the app, if you go to the editor, I don't see the ID, but if you share with me the ID, first, you have to change, for sure, the variable, the placeholder they put for the template app. Let me check if I can share my screen. Let me check if I can share my screen. Okay. Now? My Chrome crashed. Okay. Can I try to share? Okay. Let me try once more. Now it works. It works? Yeah. So, I'm here. So, where do I need to change the... So, I see that here it's app name. I guess I need to change something like that. I need to do something here with the template, right? Yeah. The app name should be changed in the other place by n8n. Do I need to do this manually or is there a way to do this? I use my ID for that. I use Visual Code. Because it's also in the EBS.yaml, which is the EBS configuration file, it's also high-fidelity. It would be good to have a command for this. You can use, if I remember correctly, find and replace, like find command. Yeah. Yeah. You can also use z or anything. Or z, yeah. Let me try. To remember. Actually, that would be a nice tool for devs. Yeah. Yeah. Okay. This will work. Okay. I thought that would work here. Let's see the other place. What? Oh, no. What is it? No, it doesn't work. What did you send? Find dot dash type f exec and then the set command that replace everywhere. You have to remove the plus in the end. What? Is it because I am in Mac? I guess so, yeah. Yeah, the set command might be different. Yeah. No, I think it's the find command. Okay. Yeah, I got it. Because I was thinking that it needs to be like this. Index format. But you are in the current path? Yeah, but maybe that's a binary or something like that. No. Oh. I guess you can do instead of dot, you can do a regex, right? Like MD? Everything finished in MD? No. Let me start it again. Give me a command. To run in Linux, to replace app name. Is app in dash name, right? Dash name. In all files my current path. Okay. I think I did something to my Git. What? I haven't done much. I cloned the repository again because I think I killed something in Git itself. But do you have the specific visual code or any UI editor? Do you use Vim? I'm just using Vim. I think you can do it with Vim, right? No. It doesn't do anything. Why? I mean, there is this DevCTL command that should do this. Anyway. Which was it? I asked it again to the TabGPP and I had the same command. Well, it's not exactly the same. Yeah, no, it's the same. I'm not sure if it's the same or not. Yeah, I think it's the same. I think set dash I is not correct because set dash I doesn't replace. Great. It just outputs. No, I should be added in place on the file. Why doesn't it work? You see? If you replace the last character with plus, which is an addition. Why do you need a semicolon? To close the exit command. This is just closing this one. So it should be fine. Oh, now it worked. Let me see. Yeah, now it worked. Why did it work now? Okay, now I can do it. There is something I guess in the directory. Oh no, now I killed my vendor. I need to do this again. I don't have it here. So find, no, rep rep, find, and I just do it one by one. Maybe as main then circle CI config pull request template and I know that's a set shell so my set shell is complaining. Oh, come on. I can only expand it. I think that doesn't work. Maybe it works. Did it work? Yeah. So what else is left? Do the rep or? The helm folder you have to do it using mb command again. Yeah. This was wrong. No. So I think it's really fine. Circle CI has been outdated too. Okay. Yeah. I need to do the vent now. I need to do this one again. Where was it? Here. And then this one needs to be the URL from that string. Oops. Okay. Yes. Yes. Okay. Yes. Okay. That should be it. Oops. You can try also now to run vendor and see what happens. Do I need to install it? Yeah. You need to install vendor first because you have to run it and you have to also have docker or something compatible to run to pull the chart. Okay. So what else to come on? It's been the sync and that's it. We will look for the config and it will fetch the Okay. Now you can see if it's there. And now you can do is and depends dependency update to see it gets the dependencies. The vendor lock also needs to Okay. But I need to add everything to get right. Yeah. Vendor lock is fine. Yeah. Then in the helm directory you can do helm dependency update so you make sure it does the right. Here in the chart I can add the dependencies you mean? You can add also the values Did I do the wrong replacement here? We didn't remove the brackets, right? Yeah. Let me just quickly add the dependencies here No, yeah. Too bad. You can search and replace which said this is easier because then you have the brackets so you can look for brackets and replace it without. Okay. Yeah. Just earlier we said or right? Or you can not target and we said the entire directory I'm not sure. Yeah. No, I mean it's it's again all these files and I'm afraid to kill my git You can push it. You can push it your chance and then I'm just I'm just going through the files one by one so it's abs-circle ci github pull request template change So I think that looks good. Yeah. What? Oh, I need to create a branch. Yeah, because the FTL Yeah. Protect the branch, right? Yeah. Okay. Pull request Should I ping you? Yes, please. Okay. Okay. Okay. Okay. Github I have not yet requested I just review Yeah. Looking at the changes looks looks fine. Have you tried the dependency update to see if it works? Uh Why do I need to do this? Like, I know that this exact version works at the moment. Because I have installed it and I don't know if I want to change anything of that setup now. Yeah, I mean, I installed it on my machine and it worked fine. Okay. Cool. Then I guess you can follow the retagging part. Like, ideally what Honeywire wants or what we want is to always retag the tools that we want. So it will be a matter of checking the container image that n8n uses and we can potentially retag it. But I think after merging this we should see the app chart already in the catalog. Do we need to add it to Github? Yes, because it will probably update something that is not up to date in the app template. But there are no other files that need to be there for Github actions or anything like that? That's through Github? The Github repository? It's through Github, yeah. Indeed, Kabbage released a Github action called update chart which does the vendor automatically for you when you create a new branch which is called update chart. It pulls from the upstream and then it updates the chart and everything and pushes it as a PR. So it's easy for you to update. Let me check if I understand this correctly. Here it says the action is deployed to the repository Giants on Github and the desired repository is set. Again, install update chart to true and run synchronize Github action. So how do I do this? I think when you add the app in the Github repository file you have to put the install update chart variable to true. I can probably see this with the Envoy gateway. Yeah, and you can ping me if you want and we can double check. I create the repository and how do I run the synchronize Github action? Synchronize Github action? Ah, when you do the PR with the new one? In Github, the repository you mean? We have to release it or something like that. I always forget that part. I think there should be a release or something. Yeah, it should be a release and then it triggers all the changes. But it should be specified in the repository. Do I need to do some retagging here? Ideally, yes. You can also trigger the synchronize workflow and it will do the trick. Releasing a new version it will do that too. I have a one-on-one. I tried to figure it out otherwise I'm going to ping you and I don't know how much time I have today anyway. Thank you very much to get where I am. Maybe it's good to put these things together to have one guide that is up to date again, I guess. Because I think it's really crucial for us if we want to test out things that those kind of steps are almost automated when we can just point something to a Helm chart and then stuff happens. That would be really, really great. But it was not painful. It was finding out some things but if we have a guide I think with Vendia and everything it's kind of easy to do. I think as soon as we release a version of your app it should be almost everything done. Within an hour you can have it working. What we can do also to test is just take a chart from AppStream, the one you selected and then say, Devin, can you do that for me and let's see if I can do all the changes or something like that. But yeah, updating this document will be also the thing to do. Let me work on that too. Thank you very much. Bye.   + +# devctl +The tool called DeepCTL in the transcript is actually called devctl and this is the repo setup command: + +Configure GitHub repository with: + + - Settings + - Permissions + - Default branch protection rules + +Usage: +  devctl repo setup [flags] REPOSITORY +  devctl repo setup [command] + +Available Commands: +  ci-webhooks Configure GitHub repository +  renovate    Enable (or disable) Renovate for the repository + +Flags: +      --allow-automerge              Allow auto-merge on pull requests, or false to forbid it. (default true) +      --allow-mergecommit            Allow merging pull requests with a merge commit, or false to prevent it. +      --allow-rebasemerge            Allow rebase-merging pull requests, or false to prevent it. +      --allow-squashmerge            Allow squash-merging pull requests, or false to prevent it. (default true) +      --allow-updatebranch           Whenever there are new changes available in the base branch, present an “update branch” option in the pull request. (default true) +      --archived                     Mark this repo as archived. +      --checks strings               Check context names for branch protection. Default will add all auto-detected checks, this can be disabled by passing an empty string. Overrides "--checks-filter" +      --checks-filter string         Provide a regex to filter checks. Checks matching the regex will be ignored. Empty string disables filter (all checks are accepted). (default "aliyun") +      --default-branch string        Default branch name (default "main") +      --delete-branch-on-merge       Automatically delete head branches when PRs are merged, or false to prevent it. (default true) +      --disable-branch-protection    Disable default branch protection +      --dry-run                      Dry-run or ready-only mode. Show what is being made but do not apply any change. +      --enable-issues                Enable issues for this repo, or false to remove them. (default true) +      --enable-projects              Enable projects for this repo, or false to remove them. +      --enable-wiki                  Enable wiki for this repo, false to remove it. +      --github-token-envvar string   Environment variable name for Github token. (default "GITHUB_TOKEN") +  -h, --help                         help for setup +      --permissions stringToString   Grant access to this repository using github_team_name=permission format. Multiple values can be provided as a comma separated list or using this flag multiple times. Permission can be one of: pull, push, admin, maintain, triage. (default [Employees=admin,bots=push]) +      --renovate                     Sets up renovate for the repo (default true) + +Global Flags: +      --log-level string   logging level (default "info") + +Use "devctl repo setup [command] --help" for more information about a command. + + +# Attachments: Git logs   +The attached files provide more details about how to setup an app repository and configure the gitops repository to deploy that application on a cluster. Especially the small changes that had to be made to get around security and compliance policies to get CI and kyverno on the cluster happy. Please analyze all the commits.  + +# Feedback on slack +In preparation for the hackathon I'd like to understand better who is using vendir for keeping charts up-to-date. Would this become our prefered way to manage upstream charts? What other ways are teams using? What is the recommendation from @honeybadger /cc +@piontec +Here is a first draft of what I'd like to do (probably needs some refinement but was generated from my creation of the n8n app recently): https://github.com/giantswarm/giantswarm/issues/32941 + +#32941 Easy and automated deployment and upgrades of apps +## Problem Statement +Currently, adding a new application (that already has a Helm chart) to a Giant Swarm Workload Cluster involves a significant number of manual steps. As evidenced by the provided transcript and git logs, the process requires developers to: +1. Manually create a repository from the app-template. +2. Configure vendir to fetch the upstream Helm chart. +3. Run devctl repo setup with correct parameters and authentication. +4. Manually (or via potentially platform-dependent scripts like sed) replace placeholder values across multiple files. +5. Manually reconcile dependencies listed in the upstream Chart.yaml with our own helm/app-name/Chart.yaml. +Show more +Assignees +@teemow +Labels +hackathon, team/planeteers +giantswarm/giantswarm | Today at 08:39 | Added by GitHub +25 replies + +Antonia + Today at 08:48 +We use vendir in rocket for a few of our charts. So far it's been working well but we still need some patches to inject things like our team label. +Mati +:no_entry: Today at 08:48 +in Cabbage we use vendir + a wrapper script that handles patches +Quentin + Today at 08:48 +Atlas uses helm chart dependencies. It has drawbacks that we cannot fix upstream without actually fixing the upstream chart so it's a bit slower sure but then we don't have to care about vendir at all (edited) +Mati +:no_entry: Today at 08:50 +we used to do vendir + upstream-repo-clone but that's not longer the case. with the sync.sh wrapper script we have a series for patches that we apply after pulling from upstream directly with vendir (edited) +Laszlo Uveges +:elephant: Today at 09:10 +My 5 cent: in Honeybadger, sometimes we use manual - e.g. flux, cos they release a single yaml file upstream with all manifests, but with KO out once, we could potentially just use flux operator maybe -, we also use a git subtree based auto update mechanism - thats a little tricky and complex from what I know about it, but its super nice when you just get a pr with the changes and might just need to solve some conflicts. +We dont use vendir, tried at a couple of projects back then and it did not cut it. Not played with chart depenencies much, we want to rather customize and extend the charts. +teemow +:sonic: Today at 09:15 +@Mati + do you have a link to the script :pray::skin-tone-2: +Mati +:no_entry: Today at 09:15 +https://github.com/giantswarm/cilium-app/tree/main/sync +:heart-8bit-1: +Marco + Today at 09:19 +The upstream repo is still useful for filing PRs, I think. (edited) +Mati +:no_entry: Today at 09:20 +yes. exaclty. we keep the upstream repo around for contributions +Marco + Today at 09:20 +So everything you proposed there and already want to use in the Giant Swarm app is also a patch in the app repo? +09:21 +Also, do not get me wrong. Didn't want to sound like "you need the upstream repo, y u no?!". :smile: +IIRC some other teams are using the upstream repo fork to build custom images in case we have more than just changes to the chart. +piontec + Today at 09:31 +adding to what Laszlo said: the script that tries to solve an update without leaving git is used for example here: https://github.com/giantswarm/zot/blob/main/.github/workflows/auto-upgrade.yaml +auto-upgrade.yaml +name: Auto upgrade the chart from upstream +on: + schedule: + - cron: "07 13 * * *" +Show more +giantswarm/zot | Added by GitHub +09:34 +We tried vendir and found it cumbersome, a lot of manual merging that otherwise can be handled by git, as both sources come from the same source repo +Pau + Today at 09:36 +at phoenix we use vendir + kustomize in some cases to generate the chart +09:39 +vendir has the problem that you can't merge files from different folders (even within the same repo) +Quentin + Today at 09:41 +It's also really painful to use and maintain when you have to maintain multiple release branches right? That was my main beef with it +Pau + Today at 09:42 +we have not faced that yet +09:42 +it's easy: don't do breaking changes :troll_parrot: +Mati +:no_entry: Today at 09:42 +we do multiple release branches without problems +Pau + Today at 09:44 +https://github.com/giantswarm/karpenter-app/blob/update-vendir/vendir.yml +https://github.com/giantswarm/karpenter-app/blob/f7bf2055ade21a70009ff9cde304751b20753f26/Makefile#L27 +here is an example of vendir + kustomize where we merge resources from 2 folders and also add annotations/labels that can't be added upstream +vendir.yml +giantswarm/karpenter-app | Added by GitHub + +Makefile +giantswarm/karpenter-app | Added by GitHub +Quentin + Today at 09:45 +Well I've done it with keda and keda supports only 3 kubernetes versions so you need to keep it up to date a lot and we have/had a lot of cluster versions :D +Pau + Today at 09:45 +yeah... the patch releases of old major are always a pain +Quentin + Today at 09:49 +Can you not do some upstream contrib so those labels are added from values? +Pau + Today at 09:51 +in some cases, the CRDs are not part of the helm chart even +Quentin + Today at 09:59 +true + +# Another transcript of Mati explaining the sync.sh script from Team Cabbage +Matías Charriere: So, let me try to share the screen. So, what we do there is we have this wrapper script where we call bender to pull the sources from the app and then apply a bunch of patches on top of that. the budgets are usually things that we cannot contribute to upstream because I don't know we add something super specific or upstream is not in favor of doing that. +Timo Derstappen: Our team labels or… +Timo Derstappen: stuff like that, right? Yeah. +Matías Charriere: Yeah, with the label is we try to use the common labels thing and that's a change that they upstream usually is willing to accept in any project adding a label it's adding a way to add labels it's okay in general but there are things like the values yes… +Timo Derstappen: But… +Timo Derstappen: but you kind of need to add it to the defaults of the value, right? Yeah. +Fernando Ripoll: the Bersian. +Matías Charriere: +Matías Charriere: but the values is always something that we change,… +Timo Derstappen: Yeah. But… +Matías Charriere: it's not something that we leave. +Timo Derstappen: but the values is something that is hard to merge, right? +Matías Charriere: So depending on the project things are little different. So that's the other problem we currently have with this method is that we don't have a centralized way to deploy the script. So we change the script depending on the project. let me see if I can share my screen because that's going to be yeah I please you can see your map scrape or… +Fernando Ripoll: I can start the screen with a script if you want. I think is here in the thread you pointed to. +Matías Charriere: It's okay. we have spreading different apps. Okay. Yeah,… +Fernando Ripoll: Where are you here? Right. +Matías Charriere: that one. Yeah, So, here we have The script is fairly simple. So we do a first stage between line 10 and 14 that we basically do a vendor sync and then a dependency update that's depending on the app also because some apps have dependencies and some apps don't have held dependencies right so we do that just to have everything clean and then we apply the patches each patch it's a +Matías Charriere: different case. Let's say some patches are just g patches where we apply the patch and… +Matías Charriere: it's a real g patch and some more patches are adding files for example or replacing files that depends on each patch. these values for example is a bit more complex. +Fernando Ripoll: For example,… +Fernando Ripoll: we can look at one, right? +Matías Charriere: We do a bunch of set you can see and we replace the file with our own values we intend to change this but yeah this is how it is now. +Fernando Ripoll: Mhm. +Matías Charriere: And we do a bunch of stuff because we are pulling ben selium from different sources from the g repo and… +Timo Derstappen: Did you take a look at the sub tree command that Bjontek did? And +Matías Charriere: from the official hem chart and then merge together that together to compose the final output. yeah yeah yeah yeah yeah we used to use sub tree and we had problems mostly on the merge strategies because some of our changes were breaking changes for upstream. +Matías Charriere: So, every time we had up an upgrade, we had issues and it's harder to track with changes because we always use commit squash merges and when you do a squash of a sub tree you lose all the information and everything is tracked inside the messages because sub tree works like that. So sub tree will track what is doing based on the commit messages and that can be a problem. If you do a squash then you need to remember not to do a squash for certain PRs. So that was our main issue with sub tree. +00:05:00 +Matías Charriere: we changed from sub tree to bender using the upstream repo which is what some teams are doing like I think it's shield is doing that but then we decided to stop using the upstream fork and then introduced this script that basically keeps everything inside the repo and from my experience it's easier to keep track of the changes and even getting rid of those changes because now I let's say that we drop the network policies patch because Art stream has it. It's just removing the folder and that's it. +Matías Charriere: you get a clean helm repo from absent. what we are also doing in this sync script is that we store the differences between what we do and upstream. This is to keep track of the changes in case we make a change that is not supposed to happen. +Matías Charriere: And it's easier for review for example because you get to guess get used to doing a review of a div of a deep that's a bit tricky… +Fernando Ripoll: Mhm. So cool. +Matías Charriere: but once you get the idea of how it works is like you can see right away what changes were introduced by upstream and that you weren't supposed to be changing let's So basically that's the whole thing. I'm not trying to sell this. I think that work for us for our team and for our repos. some teams are doing different things like building images on the upstream repo. We don't do that. We always use the upstream images for example. +Matías Charriere: So this is mostly for syncing H releases sorry H charts or teams are doing things differently because sometimes it's what they need to do right so Yeah. Yeah. +Timo Derstappen: Yeah, I would like to come up with a tool that kind of abstracts this for all the applications. +Timo Derstappen: So the applications can grow and you can have the patches and you can clean up the patches or you can add patches easily. and then yeah I mean it's specific per app and we might need to find a way to make the patches easier than using set commands or something like that to have better usability. +Timo Derstappen: But I agree that having this visible is super helpful because exactly this is something that is custom to giant one that we are adding here and do you can even put the reason in there… +Fernando Ripoll: Have you considered the option to use customize at that point or +Timo Derstappen: why is this there and stuff like that. So it's better than messing with git trees all the time and not knowing what is where and when and why. Yeah. +Matías Charriere: Yeah. Yeah. +Matías Charriere: Yeah. Yeah. we agree with I totally agree with that. I know that our team is also in line with the problem we see with customize is that you cannot template on top of that. +Timo Derstappen: Come on. +Matías Charriere: So the hem chart is a bit different of what you expect in a hem chart. I'm not sure what the case for customize is for files that are not templated in Helm. That's fine. But as soon as you need to do I don't know change the name of the deployment is going to break or… +00:10:00 +Matías Charriere: I'm not sure what other teams are doing with customer. what I saw is that you get an specific chart for your application… +Fernando Ripoll: I think that Yeah,… +Matías Charriere: but that's it. you cannot customize a lot +Fernando Ripoll: I'm not sure, but I thought that you could potentially change the template adding, for example, extra here or changing the name of a field. +Fernando Ripoll: I'm not completely sure but when I was looking at the I think it's CSI EBS chart they were using to modify some of the chart templates… +Fernando Ripoll: but yeah I didn't try Mhm. +Matías Charriere: Yeah. Yeah. +Matías Charriere: No, no, I didn't really look deep into it. I remember there was flax. I think that they were using customize at some point, but I haven't really follow it. All right,… +Fernando Ripoll: Okay. Okay. +Matías Charriere: I will jump out and… +Timo Derstappen: Yeah, we can stop it. +Matías Charriere: join my Hagaton project. let me know if you have any question. +Matías Charriere: Maybe I join tomorrow again if I finish early. Okay, bye. + +# The sync.sh script +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) ; readonly dir +cd "${dir}/.." + +# Stage 1 sync - intermediate to the ./vendir folder +set -x +vendir sync +helm dependency update helm/cilium/ +{ set +x; } 2>/dev/null + +# Patches +./sync/patches/eni/patch.sh +./sync/patches/image_registries/patch.sh +./sync/patches/readme/patch.sh +./sync/patches/networkpolicies/patch.sh +./sync/patches/values/patch.sh + +# Store diffs +rm -f ./diffs/* +for f in $(git --no-pager diff --no-exit-code --no-color --no-index vendor/cilium/install/kubernetes helm --name-only) ; do + [[ "$f" == "helm/cilium/Chart.yaml" ]] && continue + [[ "$f" == "helm/cilium/Chart.lock" ]] && continue + [[ "$f" == "helm/cilium/README.md" ]] && continue + [[ "$f" == "helm/cilium/values.schema.json" ]] && continue + [[ "$f" == "helm/cilium/values.yaml" ]] && continue + [[ "$f" =~ ^helm/cilium/charts/.* ]] && continue + + base_file="vendor/cilium/install/kubernetes/${f#"helm/"}" + [[ ! -e $base_file ]] && base_file="vendor/cilium/${f#"helm/"}" + [[ ! -e $base_file ]] && base_file="/dev/null" + + set +e + set -x + git --no-pager diff --no-exit-code --no-color --no-index "$base_file" "${f}" \ + > "./diffs/${f//\//__}.patch" # ${f//\//__} replaces all "/" with "__" + + { set +x; } 2>/dev/null + set -e + ret=$? + if [ $ret -ne 0 ] && [ $ret -ne 1 ] ; then + exit $ret + fi +done + +## How were patches generated? + +First, stage the changes (in `./helm`) and the run: + +> [!TIP] +> Skip the `-R` flags if the changes were added. + +```bash +git --no-pager diff -R helm/cilium/templates/cilium-agent/daemonset.yaml \ + > sync/patches/eni/cilium_agent__daemonset.yaml.patch +git --no-pager diff -R helm/cilium/templates/cilium-configmap.yaml \ + > sync/patches/eni/cilium-configmap.yaml.patch +``` + +## What is the patched change? + +In case something goes wrong this is the raw change: + + +In file `./helm/cilium/templates/cilium-agent/daemonset.yaml` add the env vars below to `cilium-agent` and `config` containers: + +``` + - name: CILIUM_CNI_CHAINING_MODE + valueFrom: + configMapKeyRef: + name: cilium-config + key: cni-chaining-mode + optional: true + - name: CILIUM_CUSTOM_CNI_CONF + valueFrom: + configMapKeyRef: + name: cilium-config + key: custom-cni-conf + optional: true +``` + + +In file `./helm/cilium/templates/cilium-configmap.yaml` replace: + +``` +{{- if .Values.cni.customConf }} + # legacy: v1.13 and before needed cni.customConf: true with cni.configMap + write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist +{{- end }} +``` + +with: + +``` + write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/21-cilium.conflist +``` +# Patch script +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +repo_dir=$(git rev-parse --show-toplevel) ; readonly repo_dir +script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) ; readonly script_dir + +cd "${repo_dir}" + +readonly script_dir_rel=".${script_dir#"${repo_dir}"}" + +set -x +git apply "${script_dir_rel}/cilium_agent__daemonset.yaml.patch" +git apply "${script_dir_rel}/cilium-configmap.yaml.patch" +{ set +x; } 2>/dev/null + + +# Task   +I'd like to automate this as much as possbile. To make it easy for engineers to add new apps and keep them up to date. At best this should be almost hands-free. Please create an issue for my hackathon project. Format the issue in github markdown. \ No newline at end of file From e1a833b54d266a7f7b34717d7e52f47123572714 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Fri, 28 Mar 2025 15:08:52 +0100 Subject: [PATCH 07/11] remove project info --- info.md | 345 -------------------------------------------------------- 1 file changed, 345 deletions(-) delete mode 100644 info.md diff --git a/info.md b/info.md deleted file mode 100644 index 469393aac..000000000 --- a/info.md +++ /dev/null @@ -1,345 +0,0 @@ -This is a transcript of a recording about how to install an app in a Giant Swarm Workloadcluster. The app already has a helm chart. Please analyze the transcript. - -# Transcript   -Okay, I didn't remember, but Kabbage starts using Vendir for all the… Vendir. Vendir, yeah, which we use it for the last hackathon, the Envoy Gateway, and I even like it more than the sub-module thing, the Git sub-module stuff, because, yeah, it's straightforward. So I found this one. Yes, yes, this is basically where you have to kind of say in which directory you want to include the app, right, and you can also put some constraints, like versions or whatever. If you go a bit down, you will see the example config, which looks like this config, and then you put there where it has to be the path to be updated and everything. So how they do, right, is they put the upstream chart as a dependency, and then you use it within your chart, just the dependency, and you can add, for example, in the helper the labels if you want, something like that. That's one way to do it, probably, it's better than the sub-module part. So how do I need to start? Do I need to create the transform app from the template? Yeah, yeah, you need to create first the… From the template. From the template, the repo, you can call it n8n-app, a high phone app. Yeah. Use this template here? Yeah. In our repository? Yeah. Cool, yeah. So… And then… Transform. Okay, I'm going to say if it's public or not. No, this can be public. And then, yeah. I don't think Renovate… Yeah, well, Renovate can give you, I think, updates on the charts from upstream, so it can help you, yeah. How do we describe that? Describe that? M-chart. I'm going to do M-chart. Great repository. Good. Okay. So, you can clone it. So, I can clone it locally, yeah. Okay. And… Yeah, I've cloned it. So, going back to Rendier. So, the upstream repository, I still fork it? No, you have to just put in the config. You copy, for example, the… But here, it says upstream. The upstream repository is a fork of the original repository. It holds the charts. They use the… They use the… I think this is, if you want to contribute back, to have it always a base repo. I don't think here it's necessary, because what Rendier does is, from the URL you give it, it pulls all the information, and then you just add the chart or whatever you want to update on the right path that you have put, but nothing else. So, I don't think you have to fork it, at least not right now. Okay. So, this… I need something like this here? Yeah, you need something like that. And that's a Rendier YAML inside of the repository? Let me see. Yeah, Rendier YAML, I think it is. Yeah, YAML. Oh, YAML. It's not YAML. All right. And then the vendor is… Hmm. What is that path? So, the first URL is the charts, and the included path is what you want to include from that. So, in your case, it's charts and 8n, right? If you go to the upstream. The upstream repo is here. It was… It was… mcharts, charts and 8n. No. Huh? No, no. Charts, yeah. Charts and then 8n. Yeah. Okay, yeah. Then you have to specify that path on the community chart, mchart. But… But I… Let me share this one. So, here I put the repository, and here… You can put vendor. Vendor. Vendor. Sorry, vendor. You have to put vendor. You have to put vendor in the path. No. Vendor is here. Okay, yeah. The path is… N8n? Where it's going to land. It doesn't really matter, but choose whatever you want. Because it's… You can put N8n, for example. Yeah, yeah, yeah. That makes sense. All right. Okay. Because then in the next path is where you are going to say, okay, from my local vendor N8 folder, copy everything to my helm chart template, for example. And it will update all the templates. Okay. Okay. And the second one, I don't need. Dependents, charts, dependencies. Or let me check if there is anything like that. No. All right. I only know that there are some dependencies regarding Bitnami. But I guess that's linked. But you need them. You need them. Hmm? You need… But they are linked. Yeah, but you are not copying the chart.yaml, right? Or you are. I mean, if you are copying the chart.yaml and overwriting the ask, the problem is that our chart.yaml is… It has a specific format, for example, with the log and everything. So what is doing, for example, Gavit here is copying also the charts dependency, which… Yeah, that could work. Yeah. That works. Yeah. So… But it's a different repository. It's a different repository. Yeah, but it is put in the charts folder. If you go one level up in GitHub. Here? No, it's not. Yeah. You don't have charts. Yeah. You have charts folder, right? And there you have… Ah, but it's not pull. No, no. It's… I mean, the dependency is totally external. Yeah, yeah. But sometimes they build them or push them with the dependency. No, I mean, the repository is another one, right? So if you give that to Helm, it would not pull from the same repository. It would pull from another repository. Hmm. Let me think about that, how it can be done. I guess just copying this from the chart demo, right? There's no other way around. So I copy that into our chart demo? Yeah. I mean, this is the easiest one, yeah. And I don't think dependencies change so often, right? I mean… It's… Yeah. Some maintenance might be necessary because… I mean, Vendyr is not taking care of changing the versions of the dependencies, right? For example, yeah. Yeah. It could be that Vendyr can update. No, it's not. No. I know that BNTech made a script for such cases, but using the submodels. It was trying to parse for updates and other things, but… It's not using this approach. Yeah. Okay. I think I have the Vendyr part ready, then… Then you have to do what is… Well, first, if you have just forked the app template, you have to replace the… The HEM? The HEM app variable, because otherwise it will fail when you build it. So… The template app in the redmi tells you what has to be done. Okay. I think it's just… Where is it? Ah, no. It's the intranet. It's the intranet page. It's this one. How to add? It's in the redmi, but you have to follow it. You can put it in the repo. Okay. First, I need to go to GitHub. GitHub? Yeah, you can use this. I'm giving you… Because we might need to update it in the template, too. But you can run DeepCTL to set up the repo for you instead of… Of going to… Doing it manually. Okay. So the first create and configure your repo can be replaced by DeepCTL. I will try to update this today. No, I don't know how old my DeepCTL is. Is there… Is there an update command for DeepCTL? DeepCTL has an update command. DeepCTL has an update command. Maybe my DeepCTL is too old to have the… Ah, it could be. …loading DeepCTL… …and build it. Okay. Yeah, you can do it manually if you want. It's just setting the right component. But I don't know if it's super up to date, this page, to be honest. I know that DeepCTL does for you. Because, also, you have setting, change it. And now the protection rules works a bit different. But, yeah, it should be similar. Yeah, I'm installing the binary. Yeah. Jumped from 5.15 or something like that to 7.1.2 DeepCTL. So what's the command, DeepCTL? DeepCTL repo setup and the URL. DeepCTL repo setup. DeepCTL repo setup. And then I need repo setup. How does the repository look like? It's without ATTPS. Like, that's the Antron. Only Giant Swarm, N8N app. Yeah. N variable not found. GitHub token was not found. Okay, yeah. You have to put the GitHub token variable. Otherwise, you cannot do it. Okay. I should have the token somewhere. I'm just not sure where, because I cannot find it in my environment. Let me check. I think it's... Do you have tokens generated? Yeah, let me see. I have... GitHub OPCTL token. Yeah, now it works. All right. Completed. Good. Then you can move forward. The resources, because we are using Bendy, is what is different. You can move to 3, which is in SuiteQuality. You have to check the chart.yml. Yeah. But if you just copy from the app, I think it's fine. But how to use AVS? By default, in the app, if you go to the editor, I don't see the ID, but if you share with me the ID, first, you have to change, for sure, the variable, the placeholder they put for the template app. Let me check if I can share my screen. Let me check if I can share my screen. Okay. Now? My Chrome crashed. Okay. Can I try to share? Okay. Let me try once more. Now it works. It works? Yeah. So, I'm here. So, where do I need to change the... So, I see that here it's app name. I guess I need to change something like that. I need to do something here with the template, right? Yeah. The app name should be changed in the other place by n8n. Do I need to do this manually or is there a way to do this? I use my ID for that. I use Visual Code. Because it's also in the EBS.yaml, which is the EBS configuration file, it's also high-fidelity. It would be good to have a command for this. You can use, if I remember correctly, find and replace, like find command. Yeah. Yeah. You can also use z or anything. Or z, yeah. Let me try. To remember. Actually, that would be a nice tool for devs. Yeah. Yeah. Okay. This will work. Okay. I thought that would work here. Let's see the other place. What? Oh, no. What is it? No, it doesn't work. What did you send? Find dot dash type f exec and then the set command that replace everywhere. You have to remove the plus in the end. What? Is it because I am in Mac? I guess so, yeah. Yeah, the set command might be different. Yeah. No, I think it's the find command. Okay. Yeah, I got it. Because I was thinking that it needs to be like this. Index format. But you are in the current path? Yeah, but maybe that's a binary or something like that. No. Oh. I guess you can do instead of dot, you can do a regex, right? Like MD? Everything finished in MD? No. Let me start it again. Give me a command. To run in Linux, to replace app name. Is app in dash name, right? Dash name. In all files my current path. Okay. I think I did something to my Git. What? I haven't done much. I cloned the repository again because I think I killed something in Git itself. But do you have the specific visual code or any UI editor? Do you use Vim? I'm just using Vim. I think you can do it with Vim, right? No. It doesn't do anything. Why? I mean, there is this DevCTL command that should do this. Anyway. Which was it? I asked it again to the TabGPP and I had the same command. Well, it's not exactly the same. Yeah, no, it's the same. I'm not sure if it's the same or not. Yeah, I think it's the same. I think set dash I is not correct because set dash I doesn't replace. Great. It just outputs. No, I should be added in place on the file. Why doesn't it work? You see? If you replace the last character with plus, which is an addition. Why do you need a semicolon? To close the exit command. This is just closing this one. So it should be fine. Oh, now it worked. Let me see. Yeah, now it worked. Why did it work now? Okay, now I can do it. There is something I guess in the directory. Oh no, now I killed my vendor. I need to do this again. I don't have it here. So find, no, rep rep, find, and I just do it one by one. Maybe as main then circle CI config pull request template and I know that's a set shell so my set shell is complaining. Oh, come on. I can only expand it. I think that doesn't work. Maybe it works. Did it work? Yeah. So what else is left? Do the rep or? The helm folder you have to do it using mb command again. Yeah. This was wrong. No. So I think it's really fine. Circle CI has been outdated too. Okay. Yeah. I need to do the vent now. I need to do this one again. Where was it? Here. And then this one needs to be the URL from that string. Oops. Okay. Yes. Yes. Okay. Yes. Okay. That should be it. Oops. You can try also now to run vendor and see what happens. Do I need to install it? Yeah. You need to install vendor first because you have to run it and you have to also have docker or something compatible to run to pull the chart. Okay. So what else to come on? It's been the sync and that's it. We will look for the config and it will fetch the Okay. Now you can see if it's there. And now you can do is and depends dependency update to see it gets the dependencies. The vendor lock also needs to Okay. But I need to add everything to get right. Yeah. Vendor lock is fine. Yeah. Then in the helm directory you can do helm dependency update so you make sure it does the right. Here in the chart I can add the dependencies you mean? You can add also the values Did I do the wrong replacement here? We didn't remove the brackets, right? Yeah. Let me just quickly add the dependencies here No, yeah. Too bad. You can search and replace which said this is easier because then you have the brackets so you can look for brackets and replace it without. Okay. Yeah. Just earlier we said or right? Or you can not target and we said the entire directory I'm not sure. Yeah. No, I mean it's it's again all these files and I'm afraid to kill my git You can push it. You can push it your chance and then I'm just I'm just going through the files one by one so it's abs-circle ci github pull request template change So I think that looks good. Yeah. What? Oh, I need to create a branch. Yeah, because the FTL Yeah. Protect the branch, right? Yeah. Okay. Pull request Should I ping you? Yes, please. Okay. Okay. Okay. Okay. Github I have not yet requested I just review Yeah. Looking at the changes looks looks fine. Have you tried the dependency update to see if it works? Uh Why do I need to do this? Like, I know that this exact version works at the moment. Because I have installed it and I don't know if I want to change anything of that setup now. Yeah, I mean, I installed it on my machine and it worked fine. Okay. Cool. Then I guess you can follow the retagging part. Like, ideally what Honeywire wants or what we want is to always retag the tools that we want. So it will be a matter of checking the container image that n8n uses and we can potentially retag it. But I think after merging this we should see the app chart already in the catalog. Do we need to add it to Github? Yes, because it will probably update something that is not up to date in the app template. But there are no other files that need to be there for Github actions or anything like that? That's through Github? The Github repository? It's through Github, yeah. Indeed, Kabbage released a Github action called update chart which does the vendor automatically for you when you create a new branch which is called update chart. It pulls from the upstream and then it updates the chart and everything and pushes it as a PR. So it's easy for you to update. Let me check if I understand this correctly. Here it says the action is deployed to the repository Giants on Github and the desired repository is set. Again, install update chart to true and run synchronize Github action. So how do I do this? I think when you add the app in the Github repository file you have to put the install update chart variable to true. I can probably see this with the Envoy gateway. Yeah, and you can ping me if you want and we can double check. I create the repository and how do I run the synchronize Github action? Synchronize Github action? Ah, when you do the PR with the new one? In Github, the repository you mean? We have to release it or something like that. I always forget that part. I think there should be a release or something. Yeah, it should be a release and then it triggers all the changes. But it should be specified in the repository. Do I need to do some retagging here? Ideally, yes. You can also trigger the synchronize workflow and it will do the trick. Releasing a new version it will do that too. I have a one-on-one. I tried to figure it out otherwise I'm going to ping you and I don't know how much time I have today anyway. Thank you very much to get where I am. Maybe it's good to put these things together to have one guide that is up to date again, I guess. Because I think it's really crucial for us if we want to test out things that those kind of steps are almost automated when we can just point something to a Helm chart and then stuff happens. That would be really, really great. But it was not painful. It was finding out some things but if we have a guide I think with Vendia and everything it's kind of easy to do. I think as soon as we release a version of your app it should be almost everything done. Within an hour you can have it working. What we can do also to test is just take a chart from AppStream, the one you selected and then say, Devin, can you do that for me and let's see if I can do all the changes or something like that. But yeah, updating this document will be also the thing to do. Let me work on that too. Thank you very much. Bye.   - -# devctl -The tool called DeepCTL in the transcript is actually called devctl and this is the repo setup command: - -Configure GitHub repository with: - - - Settings - - Permissions - - Default branch protection rules - -Usage: -  devctl repo setup [flags] REPOSITORY -  devctl repo setup [command] - -Available Commands: -  ci-webhooks Configure GitHub repository -  renovate    Enable (or disable) Renovate for the repository - -Flags: -      --allow-automerge              Allow auto-merge on pull requests, or false to forbid it. (default true) -      --allow-mergecommit            Allow merging pull requests with a merge commit, or false to prevent it. -      --allow-rebasemerge            Allow rebase-merging pull requests, or false to prevent it. -      --allow-squashmerge            Allow squash-merging pull requests, or false to prevent it. (default true) -      --allow-updatebranch           Whenever there are new changes available in the base branch, present an “update branch” option in the pull request. (default true) -      --archived                     Mark this repo as archived. -      --checks strings               Check context names for branch protection. Default will add all auto-detected checks, this can be disabled by passing an empty string. Overrides "--checks-filter" -      --checks-filter string         Provide a regex to filter checks. Checks matching the regex will be ignored. Empty string disables filter (all checks are accepted). (default "aliyun") -      --default-branch string        Default branch name (default "main") -      --delete-branch-on-merge       Automatically delete head branches when PRs are merged, or false to prevent it. (default true) -      --disable-branch-protection    Disable default branch protection -      --dry-run                      Dry-run or ready-only mode. Show what is being made but do not apply any change. -      --enable-issues                Enable issues for this repo, or false to remove them. (default true) -      --enable-projects              Enable projects for this repo, or false to remove them. -      --enable-wiki                  Enable wiki for this repo, false to remove it. -      --github-token-envvar string   Environment variable name for Github token. (default "GITHUB_TOKEN") -  -h, --help                         help for setup -      --permissions stringToString   Grant access to this repository using github_team_name=permission format. Multiple values can be provided as a comma separated list or using this flag multiple times. Permission can be one of: pull, push, admin, maintain, triage. (default [Employees=admin,bots=push]) -      --renovate                     Sets up renovate for the repo (default true) - -Global Flags: -      --log-level string   logging level (default "info") - -Use "devctl repo setup [command] --help" for more information about a command. - - -# Attachments: Git logs   -The attached files provide more details about how to setup an app repository and configure the gitops repository to deploy that application on a cluster. Especially the small changes that had to be made to get around security and compliance policies to get CI and kyverno on the cluster happy. Please analyze all the commits.  - -# Feedback on slack -In preparation for the hackathon I'd like to understand better who is using vendir for keeping charts up-to-date. Would this become our prefered way to manage upstream charts? What other ways are teams using? What is the recommendation from @honeybadger /cc -@piontec -Here is a first draft of what I'd like to do (probably needs some refinement but was generated from my creation of the n8n app recently): https://github.com/giantswarm/giantswarm/issues/32941 - -#32941 Easy and automated deployment and upgrades of apps -## Problem Statement -Currently, adding a new application (that already has a Helm chart) to a Giant Swarm Workload Cluster involves a significant number of manual steps. As evidenced by the provided transcript and git logs, the process requires developers to: -1. Manually create a repository from the app-template. -2. Configure vendir to fetch the upstream Helm chart. -3. Run devctl repo setup with correct parameters and authentication. -4. Manually (or via potentially platform-dependent scripts like sed) replace placeholder values across multiple files. -5. Manually reconcile dependencies listed in the upstream Chart.yaml with our own helm/app-name/Chart.yaml. -Show more -Assignees -@teemow -Labels -hackathon, team/planeteers -giantswarm/giantswarm | Today at 08:39 | Added by GitHub -25 replies - -Antonia - Today at 08:48 -We use vendir in rocket for a few of our charts. So far it's been working well but we still need some patches to inject things like our team label. -Mati -:no_entry: Today at 08:48 -in Cabbage we use vendir + a wrapper script that handles patches -Quentin - Today at 08:48 -Atlas uses helm chart dependencies. It has drawbacks that we cannot fix upstream without actually fixing the upstream chart so it's a bit slower sure but then we don't have to care about vendir at all (edited) -Mati -:no_entry: Today at 08:50 -we used to do vendir + upstream-repo-clone but that's not longer the case. with the sync.sh wrapper script we have a series for patches that we apply after pulling from upstream directly with vendir (edited) -Laszlo Uveges -:elephant: Today at 09:10 -My 5 cent: in Honeybadger, sometimes we use manual - e.g. flux, cos they release a single yaml file upstream with all manifests, but with KO out once, we could potentially just use flux operator maybe -, we also use a git subtree based auto update mechanism - thats a little tricky and complex from what I know about it, but its super nice when you just get a pr with the changes and might just need to solve some conflicts. -We dont use vendir, tried at a couple of projects back then and it did not cut it. Not played with chart depenencies much, we want to rather customize and extend the charts. -teemow -:sonic: Today at 09:15 -@Mati - do you have a link to the script :pray::skin-tone-2: -Mati -:no_entry: Today at 09:15 -https://github.com/giantswarm/cilium-app/tree/main/sync -:heart-8bit-1: -Marco - Today at 09:19 -The upstream repo is still useful for filing PRs, I think. (edited) -Mati -:no_entry: Today at 09:20 -yes. exaclty. we keep the upstream repo around for contributions -Marco - Today at 09:20 -So everything you proposed there and already want to use in the Giant Swarm app is also a patch in the app repo? -09:21 -Also, do not get me wrong. Didn't want to sound like "you need the upstream repo, y u no?!". :smile: -IIRC some other teams are using the upstream repo fork to build custom images in case we have more than just changes to the chart. -piontec - Today at 09:31 -adding to what Laszlo said: the script that tries to solve an update without leaving git is used for example here: https://github.com/giantswarm/zot/blob/main/.github/workflows/auto-upgrade.yaml -auto-upgrade.yaml -name: Auto upgrade the chart from upstream -on: - schedule: - - cron: "07 13 * * *" -Show more -giantswarm/zot | Added by GitHub -09:34 -We tried vendir and found it cumbersome, a lot of manual merging that otherwise can be handled by git, as both sources come from the same source repo -Pau - Today at 09:36 -at phoenix we use vendir + kustomize in some cases to generate the chart -09:39 -vendir has the problem that you can't merge files from different folders (even within the same repo) -Quentin - Today at 09:41 -It's also really painful to use and maintain when you have to maintain multiple release branches right? That was my main beef with it -Pau - Today at 09:42 -we have not faced that yet -09:42 -it's easy: don't do breaking changes :troll_parrot: -Mati -:no_entry: Today at 09:42 -we do multiple release branches without problems -Pau - Today at 09:44 -https://github.com/giantswarm/karpenter-app/blob/update-vendir/vendir.yml -https://github.com/giantswarm/karpenter-app/blob/f7bf2055ade21a70009ff9cde304751b20753f26/Makefile#L27 -here is an example of vendir + kustomize where we merge resources from 2 folders and also add annotations/labels that can't be added upstream -vendir.yml -giantswarm/karpenter-app | Added by GitHub - -Makefile -giantswarm/karpenter-app | Added by GitHub -Quentin - Today at 09:45 -Well I've done it with keda and keda supports only 3 kubernetes versions so you need to keep it up to date a lot and we have/had a lot of cluster versions :D -Pau - Today at 09:45 -yeah... the patch releases of old major are always a pain -Quentin - Today at 09:49 -Can you not do some upstream contrib so those labels are added from values? -Pau - Today at 09:51 -in some cases, the CRDs are not part of the helm chart even -Quentin - Today at 09:59 -true - -# Another transcript of Mati explaining the sync.sh script from Team Cabbage -Matías Charriere: So, let me try to share the screen. So, what we do there is we have this wrapper script where we call bender to pull the sources from the app and then apply a bunch of patches on top of that. the budgets are usually things that we cannot contribute to upstream because I don't know we add something super specific or upstream is not in favor of doing that. -Timo Derstappen: Our team labels or… -Timo Derstappen: stuff like that, right? Yeah. -Matías Charriere: Yeah, with the label is we try to use the common labels thing and that's a change that they upstream usually is willing to accept in any project adding a label it's adding a way to add labels it's okay in general but there are things like the values yes… -Timo Derstappen: But… -Timo Derstappen: but you kind of need to add it to the defaults of the value, right? Yeah. -Fernando Ripoll: the Bersian. -Matías Charriere: -Matías Charriere: but the values is always something that we change,… -Timo Derstappen: Yeah. But… -Matías Charriere: it's not something that we leave. -Timo Derstappen: but the values is something that is hard to merge, right? -Matías Charriere: So depending on the project things are little different. So that's the other problem we currently have with this method is that we don't have a centralized way to deploy the script. So we change the script depending on the project. let me see if I can share my screen because that's going to be yeah I please you can see your map scrape or… -Fernando Ripoll: I can start the screen with a script if you want. I think is here in the thread you pointed to. -Matías Charriere: It's okay. we have spreading different apps. Okay. Yeah,… -Fernando Ripoll: Where are you here? Right. -Matías Charriere: that one. Yeah, So, here we have The script is fairly simple. So we do a first stage between line 10 and 14 that we basically do a vendor sync and then a dependency update that's depending on the app also because some apps have dependencies and some apps don't have held dependencies right so we do that just to have everything clean and then we apply the patches each patch it's a -Matías Charriere: different case. Let's say some patches are just g patches where we apply the patch and… -Matías Charriere: it's a real g patch and some more patches are adding files for example or replacing files that depends on each patch. these values for example is a bit more complex. -Fernando Ripoll: For example,… -Fernando Ripoll: we can look at one, right? -Matías Charriere: We do a bunch of set you can see and we replace the file with our own values we intend to change this but yeah this is how it is now. -Fernando Ripoll: Mhm. -Matías Charriere: And we do a bunch of stuff because we are pulling ben selium from different sources from the g repo and… -Timo Derstappen: Did you take a look at the sub tree command that Bjontek did? And -Matías Charriere: from the official hem chart and then merge together that together to compose the final output. yeah yeah yeah yeah yeah we used to use sub tree and we had problems mostly on the merge strategies because some of our changes were breaking changes for upstream. -Matías Charriere: So, every time we had up an upgrade, we had issues and it's harder to track with changes because we always use commit squash merges and when you do a squash of a sub tree you lose all the information and everything is tracked inside the messages because sub tree works like that. So sub tree will track what is doing based on the commit messages and that can be a problem. If you do a squash then you need to remember not to do a squash for certain PRs. So that was our main issue with sub tree. -00:05:00 -Matías Charriere: we changed from sub tree to bender using the upstream repo which is what some teams are doing like I think it's shield is doing that but then we decided to stop using the upstream fork and then introduced this script that basically keeps everything inside the repo and from my experience it's easier to keep track of the changes and even getting rid of those changes because now I let's say that we drop the network policies patch because Art stream has it. It's just removing the folder and that's it. -Matías Charriere: you get a clean helm repo from absent. what we are also doing in this sync script is that we store the differences between what we do and upstream. This is to keep track of the changes in case we make a change that is not supposed to happen. -Matías Charriere: And it's easier for review for example because you get to guess get used to doing a review of a div of a deep that's a bit tricky… -Fernando Ripoll: Mhm. So cool. -Matías Charriere: but once you get the idea of how it works is like you can see right away what changes were introduced by upstream and that you weren't supposed to be changing let's So basically that's the whole thing. I'm not trying to sell this. I think that work for us for our team and for our repos. some teams are doing different things like building images on the upstream repo. We don't do that. We always use the upstream images for example. -Matías Charriere: So this is mostly for syncing H releases sorry H charts or teams are doing things differently because sometimes it's what they need to do right so Yeah. Yeah. -Timo Derstappen: Yeah, I would like to come up with a tool that kind of abstracts this for all the applications. -Timo Derstappen: So the applications can grow and you can have the patches and you can clean up the patches or you can add patches easily. and then yeah I mean it's specific per app and we might need to find a way to make the patches easier than using set commands or something like that to have better usability. -Timo Derstappen: But I agree that having this visible is super helpful because exactly this is something that is custom to giant one that we are adding here and do you can even put the reason in there… -Fernando Ripoll: Have you considered the option to use customize at that point or -Timo Derstappen: why is this there and stuff like that. So it's better than messing with git trees all the time and not knowing what is where and when and why. Yeah. -Matías Charriere: Yeah. Yeah. -Matías Charriere: Yeah. Yeah. we agree with I totally agree with that. I know that our team is also in line with the problem we see with customize is that you cannot template on top of that. -Timo Derstappen: Come on. -Matías Charriere: So the hem chart is a bit different of what you expect in a hem chart. I'm not sure what the case for customize is for files that are not templated in Helm. That's fine. But as soon as you need to do I don't know change the name of the deployment is going to break or… -00:10:00 -Matías Charriere: I'm not sure what other teams are doing with customer. what I saw is that you get an specific chart for your application… -Fernando Ripoll: I think that Yeah,… -Matías Charriere: but that's it. you cannot customize a lot -Fernando Ripoll: I'm not sure, but I thought that you could potentially change the template adding, for example, extra here or changing the name of a field. -Fernando Ripoll: I'm not completely sure but when I was looking at the I think it's CSI EBS chart they were using to modify some of the chart templates… -Fernando Ripoll: but yeah I didn't try Mhm. -Matías Charriere: Yeah. Yeah. -Matías Charriere: No, no, I didn't really look deep into it. I remember there was flax. I think that they were using customize at some point, but I haven't really follow it. All right,… -Fernando Ripoll: Okay. Okay. -Matías Charriere: I will jump out and… -Timo Derstappen: Yeah, we can stop it. -Matías Charriere: join my Hagaton project. let me know if you have any question. -Matías Charriere: Maybe I join tomorrow again if I finish early. Okay, bye. - -# The sync.sh script -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) ; readonly dir -cd "${dir}/.." - -# Stage 1 sync - intermediate to the ./vendir folder -set -x -vendir sync -helm dependency update helm/cilium/ -{ set +x; } 2>/dev/null - -# Patches -./sync/patches/eni/patch.sh -./sync/patches/image_registries/patch.sh -./sync/patches/readme/patch.sh -./sync/patches/networkpolicies/patch.sh -./sync/patches/values/patch.sh - -# Store diffs -rm -f ./diffs/* -for f in $(git --no-pager diff --no-exit-code --no-color --no-index vendor/cilium/install/kubernetes helm --name-only) ; do - [[ "$f" == "helm/cilium/Chart.yaml" ]] && continue - [[ "$f" == "helm/cilium/Chart.lock" ]] && continue - [[ "$f" == "helm/cilium/README.md" ]] && continue - [[ "$f" == "helm/cilium/values.schema.json" ]] && continue - [[ "$f" == "helm/cilium/values.yaml" ]] && continue - [[ "$f" =~ ^helm/cilium/charts/.* ]] && continue - - base_file="vendor/cilium/install/kubernetes/${f#"helm/"}" - [[ ! -e $base_file ]] && base_file="vendor/cilium/${f#"helm/"}" - [[ ! -e $base_file ]] && base_file="/dev/null" - - set +e - set -x - git --no-pager diff --no-exit-code --no-color --no-index "$base_file" "${f}" \ - > "./diffs/${f//\//__}.patch" # ${f//\//__} replaces all "/" with "__" - - { set +x; } 2>/dev/null - set -e - ret=$? - if [ $ret -ne 0 ] && [ $ret -ne 1 ] ; then - exit $ret - fi -done - -## How were patches generated? - -First, stage the changes (in `./helm`) and the run: - -> [!TIP] -> Skip the `-R` flags if the changes were added. - -```bash -git --no-pager diff -R helm/cilium/templates/cilium-agent/daemonset.yaml \ - > sync/patches/eni/cilium_agent__daemonset.yaml.patch -git --no-pager diff -R helm/cilium/templates/cilium-configmap.yaml \ - > sync/patches/eni/cilium-configmap.yaml.patch -``` - -## What is the patched change? - -In case something goes wrong this is the raw change: - - -In file `./helm/cilium/templates/cilium-agent/daemonset.yaml` add the env vars below to `cilium-agent` and `config` containers: - -``` - - name: CILIUM_CNI_CHAINING_MODE - valueFrom: - configMapKeyRef: - name: cilium-config - key: cni-chaining-mode - optional: true - - name: CILIUM_CUSTOM_CNI_CONF - valueFrom: - configMapKeyRef: - name: cilium-config - key: custom-cni-conf - optional: true -``` - - -In file `./helm/cilium/templates/cilium-configmap.yaml` replace: - -``` -{{- if .Values.cni.customConf }} - # legacy: v1.13 and before needed cni.customConf: true with cni.configMap - write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/05-cilium.conflist -{{- end }} -``` - -with: - -``` - write-cni-conf-when-ready: {{ .Values.cni.hostConfDirMountPath }}/21-cilium.conflist -``` -# Patch script -#!/usr/bin/env bash - -set -o errexit -set -o nounset -set -o pipefail - -repo_dir=$(git rev-parse --show-toplevel) ; readonly repo_dir -script_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) ; readonly script_dir - -cd "${repo_dir}" - -readonly script_dir_rel=".${script_dir#"${repo_dir}"}" - -set -x -git apply "${script_dir_rel}/cilium_agent__daemonset.yaml.patch" -git apply "${script_dir_rel}/cilium-configmap.yaml.patch" -{ set +x; } 2>/dev/null - - -# Task   -I'd like to automate this as much as possbile. To make it easy for engineers to add new apps and keep them up to date. At best this should be almost hands-free. Please create an issue for my hackathon project. Format the issue in github markdown. \ No newline at end of file From 07acd0d1d3134a3deaa3961cd9892daff0882096 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Fri, 28 Mar 2025 20:18:48 +0100 Subject: [PATCH 08/11] Refactor bootstrap flag handling and improve error checking; update default methods and file permissions --- cmd/app/bootstrap/flag.go | 36 +++++++++++++++++++++++++----------- cmd/app/bootstrap/runner.go | 32 +++++++++++++++++++------------- 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/cmd/app/bootstrap/flag.go b/cmd/app/bootstrap/flag.go index 1b5f2de84..1ded0509b 100644 --- a/cmd/app/bootstrap/flag.go +++ b/cmd/app/bootstrap/flag.go @@ -12,8 +12,13 @@ const ( flagTeam = "team" flagSyncMethod = "sync-method" flagPatchMethod = "patch-method" - flagGithubToken = "github-token-envvar" + flagGithubToken = "github-token-envvar" // #nosec G101 flagDryRun = "dry-run" + + // Method types + methodKustomize = "kustomize" + methodVendir = "vendir" + methodScript = "script" ) type flag struct { @@ -32,15 +37,24 @@ func (f *flag) Init(cmd *cobra.Command) { cmd.Flags().StringVar(&f.UpstreamRepo, flagUpstreamRepo, "", "URL of the upstream repository containing the Helm chart") cmd.Flags().StringVar(&f.UpstreamChart, flagUpstreamChart, "", "Path to the Helm chart in the upstream repository") cmd.Flags().StringVar(&f.Team, flagTeam, "", "Team responsible for the app") - cmd.Flags().StringVar(&f.SyncMethod, flagSyncMethod, "vendir", "Method to sync upstream chart (vendir or kustomize)") - cmd.Flags().StringVar(&f.PatchMethod, flagPatchMethod, "script", "Method to patch upstream chart (script or kustomize)") + cmd.Flags().StringVar(&f.SyncMethod, flagSyncMethod, methodVendir, "Method to sync upstream chart (vendir or kustomize)") + cmd.Flags().StringVar(&f.PatchMethod, flagPatchMethod, methodScript, "Method to patch upstream chart (script or kustomize)") cmd.Flags().StringVar(&f.GithubToken, flagGithubToken, "GITHUB_TOKEN", "Name of environment variable containing GitHub token") cmd.Flags().BoolVar(&f.DryRun, flagDryRun, false, "If set, only print what would be done") - cmd.MarkFlagRequired(flagName) - cmd.MarkFlagRequired(flagUpstreamRepo) - cmd.MarkFlagRequired(flagUpstreamChart) - cmd.MarkFlagRequired(flagTeam) + // Check errors from MarkFlagRequired + if err := cmd.MarkFlagRequired(flagName); err != nil { + panic(err) + } + if err := cmd.MarkFlagRequired(flagUpstreamRepo); err != nil { + panic(err) + } + if err := cmd.MarkFlagRequired(flagUpstreamChart); err != nil { + panic(err) + } + if err := cmd.MarkFlagRequired(flagTeam); err != nil { + panic(err) + } } func (f *flag) Validate() error { @@ -56,11 +70,11 @@ func (f *flag) Validate() error { if f.Team == "" { return microerror.Maskf(invalidFlagError, "--%s must not be empty", flagTeam) } - if f.SyncMethod != "vendir" && f.SyncMethod != "kustomize" { - return microerror.Maskf(invalidFlagError, "--%s must be either 'vendir' or 'kustomize'", flagSyncMethod) + if f.SyncMethod != methodVendir && f.SyncMethod != methodKustomize { + return microerror.Maskf(invalidFlagError, "--%s must be either '%s' or '%s'", flagSyncMethod, methodVendir, methodKustomize) } - if f.PatchMethod != "script" && f.PatchMethod != "kustomize" { - return microerror.Maskf(invalidFlagError, "--%s must be either 'script' or 'kustomize'", flagPatchMethod) + if f.PatchMethod != methodScript && f.PatchMethod != methodKustomize { + return microerror.Maskf(invalidFlagError, "--%s must be either '%s' or '%s'", flagPatchMethod, methodScript, methodKustomize) } return nil diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index 58ce98336..ac11b3e7b 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -21,6 +21,12 @@ import ( "github.com/giantswarm/devctl/v7/pkg/githubclient" ) +const ( + logLevelDebug = "debug" + fileMode0600 = 0600 + fileMode0755 = 0755 +) + type runner struct { flag *flag logger micrologger.Logger @@ -189,7 +195,7 @@ func (r *runner) createRepository(ctx context.Context, name string, owner string // Create a logger that only outputs in debug mode logger := logrus.New() - if os.Getenv("LOG_LEVEL") == "debug" { + if os.Getenv("LOG_LEVEL") == logLevelDebug { logger.SetOutput(r.stdout) } else { logger.SetOutput(io.Discard) @@ -208,9 +214,9 @@ func (r *runner) createRepository(ctx context.Context, name string, owner string repoName := fmt.Sprintf("%s-app", name) repo := &github.Repository{ - Name: github.String(repoName), - Private: github.Bool(false), - Description: github.String(fmt.Sprintf("Helm chart for %s", name)), + Name: github.Ptr(repoName), + Private: github.Ptr(false), + Description: github.Ptr(fmt.Sprintf("Helm chart for %s", name)), } _, err = client.CreateFromTemplate(ctx, owner, templateRepo, owner, repo) @@ -303,7 +309,7 @@ directories: r.flag.Name, r.flag.UpstreamChart) - err := os.WriteFile(filepath.Join(repoPath, "vendir.yml"), []byte(vendirConfig), 0644) + err := os.WriteFile(filepath.Join(repoPath, "vendir.yml"), []byte(vendirConfig), fileMode0600) if err != nil { return microerror.Mask(err) } @@ -337,7 +343,7 @@ func (r *runner) configurePatchScript(ctx context.Context, repoPath string) erro // Create sync directory and patches subdirectory syncDir := filepath.Join(repoPath, "sync") patchesDir := filepath.Join(syncDir, "patches") - err := os.MkdirAll(patchesDir, 0755) + err := os.MkdirAll(patchesDir, fileMode0755) if err != nil { return microerror.Mask(err) } @@ -394,7 +400,7 @@ done` r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name, r.flag.Name) - err = os.WriteFile(filepath.Join(syncDir, "sync.sh"), []byte(syncScript), 0755) + err = os.WriteFile(filepath.Join(syncDir, "sync.sh"), []byte(syncScript), fileMode0755) if err != nil { return microerror.Mask(err) } @@ -679,7 +685,7 @@ func (r *runner) createGithubRepoPR(ctx context.Context) (error, string) { return microerror.Mask(err), "" } - err = os.WriteFile(teamFile, buf.Bytes(), 0644) + err = os.WriteFile(teamFile, buf.Bytes(), fileMode0600) if err != nil { return microerror.Mask(err), "" } @@ -706,11 +712,11 @@ func (r *runner) createGithubRepoPR(ctx context.Context) (error, string) { prBody := fmt.Sprintf("This PR adds the %s-app to the team-%s repositories configuration.", r.flag.Name, r.flag.Team) pr := &github.NewPullRequest{ - Title: github.String(prTitle), - Head: github.String(branchName), - Base: github.String("main"), - Body: github.String(prBody), - MaintainerCanModify: github.Bool(true), + Title: github.Ptr(prTitle), + Head: github.Ptr(branchName), + Base: github.Ptr("main"), + Body: github.Ptr(prBody), + MaintainerCanModify: github.Ptr(true), } createdPR, err := client.CreatePullRequest(ctx, "giantswarm", "github", pr) From ccdb4b0ffe3739bf4ee674f9d9845061e6724ce0 Mon Sep 17 00:00:00 2001 From: Timo Derstappen Date: Fri, 28 Mar 2025 20:22:08 +0100 Subject: [PATCH 09/11] Refactor debug logging checks and update GitHub client repository owner assignment --- cmd/app/bootstrap/runner.go | 8 ++++---- pkg/githubclient/client_repository.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index ac11b3e7b..db14d6dfa 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -488,7 +488,7 @@ func (r *runner) setupCICD(ctx context.Context, repoPath string) error { cmd.Dir = repoPath // Only show output in debug mode - if os.Getenv("LOG_LEVEL") == "debug" { + if os.Getenv("LOG_LEVEL") == logLevelDebug { cmd.Stdout = r.stdout cmd.Stderr = r.stderr } @@ -535,7 +535,7 @@ func (r *runner) enableBranchProtection(ctx context.Context, repoPath string) er cmd.Dir = repoPath // Only show output in debug mode - if os.Getenv("LOG_LEVEL") == "debug" { + if os.Getenv("LOG_LEVEL") == logLevelDebug { cmd.Stdout = r.stdout cmd.Stderr = r.stderr } @@ -557,7 +557,7 @@ func (r *runner) execCommand(ctx context.Context, dir string, command string, ar } // Only show command output in debug mode - if os.Getenv("LOG_LEVEL") == "debug" { + if os.Getenv("LOG_LEVEL") == logLevelDebug { cmd.Stdout = r.stdout cmd.Stderr = r.stderr } @@ -593,7 +593,7 @@ func (r *runner) createGithubRepoPR(ctx context.Context) (error, string) { // Create a logger that only outputs in debug mode logger := logrus.New() - if os.Getenv("LOG_LEVEL") == "debug" { + if os.Getenv("LOG_LEVEL") == logLevelDebug { logger.SetOutput(r.stdout) } else { logger.SetOutput(io.Discard) diff --git a/pkg/githubclient/client_repository.go b/pkg/githubclient/client_repository.go index 1a5b9ee69..1798970a5 100644 --- a/pkg/githubclient/client_repository.go +++ b/pkg/githubclient/client_repository.go @@ -449,7 +449,7 @@ func (c *Client) CreateFromTemplate(ctx context.Context, templateOwner, template req := &github.TemplateRepoRequest{ Name: repository.Name, - Owner: github.String(newOwner), + Owner: github.Ptr(newOwner), Description: repository.Description, Private: repository.Private, } From dba0078638b2dfddb3122fc379fd7f2031a5eef2 Mon Sep 17 00:00:00 2001 From: pipo02mix Date: Thu, 3 Apr 2025 16:34:53 +0200 Subject: [PATCH 10/11] Adapt to mac --- cmd/app/bootstrap/runner.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index db14d6dfa..9cdd5848a 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -314,6 +314,8 @@ directories: return microerror.Mask(err) } + fmt.Printf("Repo path %s \n vendir.yml file created \n", repoPath) + // Run initial sync err = r.execCommand(ctx, repoPath, "vendir", "sync") if err != nil { @@ -422,22 +424,24 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error return microerror.Mask(err) } + replaceString := fmt.Sprintf("s|github.com/giantswarm/{APP-NAME}|github.com/giantswarm/%s-app|g", r.flag.Name) // First replace GitHub URLs that need the -app suffix err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "f", - "-not", "-path", "./.git/*", - "-exec", "sed", "-i", - fmt.Sprintf("s|github.com/giantswarm/{APP-NAME}|github.com/giantswarm/%s-app|g", r.flag.Name), + "find", ".", "-type", "d", "-name", ".git", + "-prune", "-o", "-type", "f", + "-exec", "sed", "-i ''", + replaceString, "{}", "+") if err != nil { + fmt.Printf("Error replacing GitHub URLs: repo path %s \n command: find . -type d -name -path .git -prune -o -type f -exec sed -i %s {} +\n", repoPath, replaceString) return microerror.Mask(err) } // Then replace CircleCI URLs that need the -app suffix err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "f", - "-not", "-path", "./.git/*", - "-exec", "sed", "-i", + "find", ".", "-type", "d", "-name", ".git", + "-prune", "-o", "-type", "f", + "-exec", "sed", "-i ''", fmt.Sprintf("s|gh/giantswarm/{APP-NAME}/|gh/giantswarm/%s-app/|g", r.flag.Name), "{}", "+") if err != nil { @@ -446,16 +450,16 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error // Then do the general replacement for all other cases err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "f", - "-not", "-path", "./.git/*", - "-exec", "sed", "-i", fmt.Sprintf("s/{APP-NAME}/%s/g", r.flag.Name), "{}", "+") + "find", ".", "-type", "d", "-name", ".git", + "-prune", "-o", "-type", "f", + "-exec", "sed", "-i ''", fmt.Sprintf("s/{APP-NAME}/%s/g", r.flag.Name), "{}", "+") if err != nil { return microerror.Mask(err) } // Replace team in CODEOWNERS err = r.execCommand(ctx, repoPath, - "sed", "-i", + "sed", "-i ''", fmt.Sprintf("s/@giantswarm\\/team-honeybadger/@giantswarm\\/team-%s/g", r.flag.Team), "CODEOWNERS") if err != nil { @@ -464,7 +468,7 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error // Add team label err = r.execCommand(ctx, repoPath, - "sed", "-i", + "sed", "-i ''", fmt.Sprintf(`s/app.kubernetes.io\/name: %s/app.kubernetes.io\/name: %s\n application.giantswarm.io\/team: %s/`, r.flag.Name, r.flag.Name, r.flag.Team), fmt.Sprintf("helm/%s/templates/_helpers.tpl", r.flag.Name)) From 02e98ee56e64c0906328d8adea304e27ec04f135 Mon Sep 17 00:00:00 2001 From: pipo02mix Date: Mon, 7 Apr 2025 15:38:46 +0200 Subject: [PATCH 11/11] Make it Mac compatible --- cmd/app/bootstrap/runner.go | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/cmd/app/bootstrap/runner.go b/cmd/app/bootstrap/runner.go index 9cdd5848a..0d0eef94e 100644 --- a/cmd/app/bootstrap/runner.go +++ b/cmd/app/bootstrap/runner.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "time" "github.com/briandowns/spinner" @@ -424,25 +425,29 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error return microerror.Mask(err) } - replaceString := fmt.Sprintf("s|github.com/giantswarm/{APP-NAME}|github.com/giantswarm/%s-app|g", r.flag.Name) + // Determine the correct sed -i flag based on the OS + var sedInPlaceFlag string + if runtime.GOOS == "darwin" { + sedInPlaceFlag = "sed -i ''" // macOS requires an empty string after -i + } else { + sedInPlaceFlag = "sed -i" // Linux does not require an empty string + } + + replaceString := fmt.Sprintf("\"s|github.com/giantswarm/{APP-NAME}|github.com/giantswarm/%s-app|g\"", r.flag.Name) // First replace GitHub URLs that need the -app suffix err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "d", "-name", ".git", - "-prune", "-o", "-type", "f", - "-exec", "sed", "-i ''", - replaceString, - "{}", "+") + "find", ".", "-path", "\"./.git\"", "-prune", + "-o", "-type", "f", "-exec", sedInPlaceFlag, + replaceString, "{}", "+") if err != nil { - fmt.Printf("Error replacing GitHub URLs: repo path %s \n command: find . -type d -name -path .git -prune -o -type f -exec sed -i %s {} +\n", repoPath, replaceString) return microerror.Mask(err) } // Then replace CircleCI URLs that need the -app suffix err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "d", "-name", ".git", - "-prune", "-o", "-type", "f", - "-exec", "sed", "-i ''", - fmt.Sprintf("s|gh/giantswarm/{APP-NAME}/|gh/giantswarm/%s-app/|g", r.flag.Name), + "find", ".", "-path", "'./.git'", "-prune", + "-o", "-type", "f", "-exec", sedInPlaceFlag, + fmt.Sprintf("\"s|gh/giantswarm/{APP-NAME}/|gh/giantswarm/%s-app/|g\"", r.flag.Name), "{}", "+") if err != nil { return microerror.Mask(err) @@ -450,16 +455,16 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error // Then do the general replacement for all other cases err = r.execCommand(ctx, repoPath, - "find", ".", "-type", "d", "-name", ".git", - "-prune", "-o", "-type", "f", - "-exec", "sed", "-i ''", fmt.Sprintf("s/{APP-NAME}/%s/g", r.flag.Name), "{}", "+") + "find", ".", "-path", "'./.git'", "-prune", + "-o", "-type", "f", "-exec", sedInPlaceFlag, + fmt.Sprintf("\"s/{APP-NAME}/%s/g\"", r.flag.Name), "{}", "+") if err != nil { return microerror.Mask(err) } // Replace team in CODEOWNERS err = r.execCommand(ctx, repoPath, - "sed", "-i ''", + sedInPlaceFlag, fmt.Sprintf("s/@giantswarm\\/team-honeybadger/@giantswarm\\/team-%s/g", r.flag.Team), "CODEOWNERS") if err != nil { @@ -468,7 +473,7 @@ func (r *runner) replacePlaceholders(ctx context.Context, repoPath string) error // Add team label err = r.execCommand(ctx, repoPath, - "sed", "-i ''", + sedInPlaceFlag, fmt.Sprintf(`s/app.kubernetes.io\/name: %s/app.kubernetes.io\/name: %s\n application.giantswarm.io\/team: %s/`, r.flag.Name, r.flag.Name, r.flag.Team), fmt.Sprintf("helm/%s/templates/_helpers.tpl", r.flag.Name))