diff --git a/.govulncheck.yaml b/.govulncheck.yaml index cc1647dcd..fb027e09c 100644 --- a/.govulncheck.yaml +++ b/.govulncheck.yaml @@ -1,79 +1 @@ -ignored-vulnerabilities: - # Incorrect parsing of IPv6 host literals in net/url - # Found in: net/url@go1.24.13 - # Fixed in: net/url@go1.25.8 - - id: GO-2026-4601 - info: https://pkg.go.dev/vuln/GO-2026-4601 - silence-until: 2026-08-31 - # FileInfo can escape from a Root in os - # Found in: os@go1.24.13 - # Fixed in: os@go1.25.8 - - id: GO-2026-4602 - info: https://pkg.go.dev/vuln/GO-2026-4602 - silence-until: 2026-08-31 - # Unauthenticated TLS 1.3 KeyUpdate record can cause persistent connection retention and DoS in crypto/tls - # Found in: crypto/tls@go1.24.13 - # Fixed in: crypto/tls@go1.25.9 - - id: GO-2026-4870 - info: https://pkg.go.dev/vuln/GO-2026-4870 - silence-until: 2026-08-31 - # Inefficient policy validation in crypto/x509 - # Found in: crypto/x509@go1.24.13 - # Fixed in: crypto/x509@go1.25.9 - - id: GO-2026-4946 - info: https://pkg.go.dev/vuln/GO-2026-4946 - silence-until: 2026-08-31 - # Infinite loop in HTTP/2 transport when given bad SETTINGS_MAX_FRAME_SIZE in net/http/internal/http2 in golang.org/x/net - # Found in: golang.org/x/net/http2@v0.47.0 - # Fixed in: golang.org/x/net/http2@v0.53.0 - - id: GO-2026-4918 - info: https://pkg.go.dev/vuln/GO-2026-4918 - silence-until: 2026-08-31 - # Unexpected work during chain building in crypto/x509 - # Found in: crypto/x509@go1.24.13 - # Fixed in: crypto/x509@go1.25.9 - - id: GO-2026-4947 - info: https://pkg.go.dev/vuln/GO-2026-4947 - silence-until: 2026-08-31 - # Panic in Dial and LookupPort when handling NUL byte on Windows in net - # Found in: net@go1.24.13 - # Fixed in: net@go1.25.10 - - id: GO-2026-4971 - info: https://pkg.go.dev/vuln/GO-2026-4971 - silence-until: 2026-08-31 - # Invoking failure to reject ASCII-only Punycode-encoded labels in golang.org/x/net/idna - # Found in: golang.org/x/net/idna@v0.47.0 - # Fixed in: golang.org/x/net/idna@v0.55.0 - - id: GO-2026-5026 - info: https://pkg.go.dev/vuln/GO-2026-5026 - silence-until: 2026-08-31 - # Inefficient candidate hostname parsing in crypto/x509 - # Found in: crypto/x509@go1.24.13 - # Fixed in: crypto/x509@go1.25.11 - - id: GO-2026-5037 - info: https://pkg.go.dev/vuln/GO-2026-5037 - silence-until: 2026-08-31 - # Quadratic complexity in WordDecoder.DecodeHeader in mime - # Found in: mime@go1.24.13 - # Fixed in: mime@go1.25.11 - - id: GO-2026-5038 - info: https://pkg.go.dev/vuln/GO-2026-5038 - silence-until: 2026-08-31 - # Arbitrary inputs are included in errors without any escaping in net/textproto - # Found in: net/textproto@go1.24.13 - # Fixed in: net/textproto@go1.25.11 - - id: GO-2026-5039 - info: https://pkg.go.dev/vuln/GO-2026-5039 - silence-until: 2026-08-31 - # Invoking Encrypted Client Hello privacy leak in crypto/tls - # Found in: crypto/tls@go1.24.13 - # Fixed in: crypto/tls@go1.25.12 - - id: GO-2026-5856 - info: https://pkg.go.dev/vuln/GO-2026-5856 - silence-until: 2026-08-31 - # Infinite loop on invalid input in golang.org/x/text - # Found in: golang.org/x/text/unicode/norm@v0.31.0 - # Fixed in: golang.org/x/text/unicode/norm@v0.39.0 - - id: GO-2026-5970 - info: https://pkg.go.dev/vuln/GO-2026-5970 - silence-until: 2026-08-31 +ignored-vulnerabilities: [] diff --git a/README.adoc b/README.adoc deleted file mode 100644 index a96e4abbf..000000000 --- a/README.adoc +++ /dev/null @@ -1,175 +0,0 @@ -= CodeReady Toolchain E2E Tests - -image:https://goreportcard.com/badge/github.com/codeready-toolchain/toolchain-e2e[Go Report Card, link="https://goreportcard.com/report/github.com/codeready-toolchain/toolchain-e2e"] -image:https://godoc.org/github.com/codeready-toolchain/toolchain-e2e?status.png[GoDoc,link="https://godoc.org/github.com/codeready-toolchain/toolchain-e2e"] - -This repo contains e2e tests for https://github.com/codeready-toolchain/host-operator[host] and https://github.com/codeready-toolchain/member-operator[member] operators of CodeReady Toolchain. - -== Build - -Requires Go version 1.24.x (1.24.13 or higher) - download for your development environment https://golang.org/dl/[here]. - -This repository uses https://github.com/golang/go/wiki/Modules[Go modules]. - -== Step by step guide - running in CodeReady Containers - -Refer to link:openshift_local.adoc[this guide] for detailed instructions on running the e2e tests in a local CodeReady Containers cluster. - -== End-to-End Tests - -The e2e tests are executed against host and member operators running in OpenShift. The operators are built from the https://github.com/codeready-toolchain/host-operator[host-operator] and https://github.com/codeready-toolchain/member-operator[member-operator] repositories. - -Since the changes in the e2e repo sometimes require changes in some of the operator repositories at the same time, the logic that executes the e2e tests supports a feature of pairing the e2e PR with *one* other PR based on branch names. -Before the e2e tests are executed in openshift-ci, the logic automatically tries to pair a PR opened for this (toolchain-e2e) repository with a branch of the same name that potentially could exist in any of the developer's fork of the operator repositories. - -For example, if a developer with GH account `cooljohn` opens a PR (for toolchain-e2e repo) from a branch `fix-reconcile`, then the logic checks if there is a branch `fix-reconcile` also in the `cooljohn/host-operator` and `cooljohn/member-operator` forks. -Let's say that `cooljohn/host-operator` contains such a branch but `cooljohn/member-operator` doesn't, then the logic: - -1. clones latest changes of both repos https://github.com/codeready-toolchain/host-operator[codeready-toolchain/host-operator] and https://github.com/codeready-toolchain/member-operator[codeready-toolchain/member-operator] -2. fetches the `fix-reconcile` branch from `cooljohn/host-operator` fork -3. merges `master` branch with the changes from `fix-reconcile` branch inside of `host-operator` repo -4. builds images from the merge branch of `host-operator` repo and from `master` branch of `member-operator` repo & deploys them to OpenShift -5. runs e2e tests taken from the opened PR - -It would work analogically also for the case when none of the repositories contain the branch name. However, if *both* repositories contain the branch name then it will result in an error. -This is by design because OLM does not ensure that both the host-operator and member-operator will be updated at the same time in production. Prohibiting PR pairing with more than one repo helps ensure they do not depend on one another and can be updated separately. - -If you still don't know what to do with e2e tests in some use-cases, go to <> section where all use-cases are covered. - -NOTE: You can also pair a toolchain-e2e PR with a PR from the https://github.com/kubesaw/ksctl repository, but this is one-directional only (e2e tests are not yet executed in the `ksctl` repo). To rely on a locally installed version of the `ksctl` binary, set `USE_INSTALLED_KSCTL=true`. If you want to build and use the `ksctl` binary from your local `kubesaw/ksctl` repository, set the `KSCTL_REPO_PATH` variable to point to your `kubesaw/ksctl` folder. - -=== Running Locally - -*Prerequisites*: - -* Install the link:required_tools.adoc[required tools]. -* Configure link:quay.adoc[your quay account for dev deployment]. - -==== Running in a Development Environment - -See the procedure to install the Dev Sandbox in a development environment https://kubesaw.github.io/contributing/[here]. - -=== Running End-to-End Tests - -. link:quay.adoc[Configure your quay account for dev deployment] -Although the e2e tests are in the separated repository than the actual operators are, it's still possible to run them against the current code that is at HEAD of the operator repositories. -There are multiple Makefile targets that will execute the e2e tests, they just differ in where the operators' code is taken from: - -* `make test-e2e` - this target clones the latest changes from both repos https://github.com/codeready-toolchain/host-operator[host-operator] and https://github.com/codeready-toolchain/member-operator[member-operator], builds images from the master, deploys to OpenShift and runs e2e tests against them. -* `make test-e2e-local` - this target doesn't clone anything, but it builds operator images from the directories `../host-operator` and `../member-operator`. These images deploys to OpenShift and runs e2e tests against them. -* `make test-e2e-member-local` - this target clones only the https://github.com/codeready-toolchain/host-operator[host-operator] repo and builds an image from it. For member-operator, it builds the image from `../member-operator` directory. These images deploys to OpenShift and runs e2e tests against them. -* `make test-e2e-host-local` - this target clones only the https://github.com/codeready-toolchain/member-operator[member-operator] repo and builds an image from it. For host-operator, it builds the image from `../host-operator` directory. These images deploys to OpenShift and runs e2e tests against them. - -The e2e tests will take care of creating all needed namespaces with random names (or see below for enforcing some specific namespace names). -It will also create all required CRDs, role and role bindings for the service accounts, build the container images for both operators and push them to the OpenShift container registry. Finally, it will deploy the operators and run the tests using the operator-sdk. - - -NOTE: you can override the default namespace names where the end-to-end tests are going to be executed - eg.: `make test-e2e HOST_NS=my-host MEMBER_NS=my-member` file. - -NOTE: you can disable SSL/TLS certificate verification in tests setting the `DISABLE_KUBE_CLIENT_TLS_VERIFY` variable to `true` - eg.: `make test-e2e DISABLE_KUBE_CLIENT_TLS_VERIFY=true`. This flag helps when you test in clusters using Self-Signed Certificates. - -NOTE: you can specify a regular expression to selectively run particular test cases by setting the `TESTS_RUN_FILTER_REGEXP` variable. eg.: `make test-e2e TESTS_RUN_FILTER_REGEXP="TestSetupMigration"`. For more information see the https://pkg.go.dev/cmd/go#hdr-Testing_flags[go test -run documentation]. - -NOTE: you should not override `SECOND_MEMBER_MODE` in test-e2e, since the e2e tests require a second member operator. - -=== Running/Debugging e2e tests from your IDE - -In order to run/debug tests from your IDE you'll need to export some required env variables, those will be used by the test framework to interact with the operator namespaces and the other toolchain resources in you cluster. -Following snippet of code should be TEMPORARILY added at the top of the test you want to run/debug from your IDE: - -``` -os.Setenv("MEMBER_NS","toolchain-member-18161051") -// `SECOND_MEMBER_MODE` should be set to true, since the e2e tests require a second member operator. -os.Setenv("SECOND_MEMBER_MODE","true") -os.Setenv("MEMBER_NS_2","toolchain-member2-18161051") -os.Setenv("HOST_NS","toolchain-host-18161051") -os.Setenv("REGISTRATION_SERVICE_NS","toolchain-host-18161051") -os.Setenv("KUBECONFIG", "~/aws-cluster-test/my-devsandbox/auth/kubeconfig") -``` - -example of Test case code containing the debugging env variables: -``` -package parallel - -import ( - "context" - "os" - "testing" -) - -func TestCreateSpaceRequest(t *testing.T) { - os.Setenv("MEMBER_NS","toolchain-member-18161051") - os.Setenv("SECOND_MEMBER_MODE","true") - os.Setenv("MEMBER_NS_2","toolchain-member2-18161051") - os.Setenv("HOST_NS","toolchain-host-18161051") - os.Setenv("REGISTRATION_SERVICE_NS","toolchain-host-18161051") - os.Setenv("KUBECONFIG", "~/aws-cluster-test/my-devsandbox/auth/kubeconfig") - // some more code here ... - - t.Run("create space request", func(t *testing.T) { - // test case implementation here ... -.... -``` - -NOTE: replace the values with the ones from your dev/test environment and REMEMBER TO REMOVE THE SNIPPET BEFORE COMMITTING THE CODE OR OPENING A PR IN GH :) - - -===== What To Do - -If you are still confused by the different e2e/operator location, execution and branch pairing, see the following cases and needed steps: - -* *Working locally:* -** *Need to verify changes in e2e tests against the latest version of both operators:* -*** run `make test-e2e` -** *You are working in both repos `toolchain-e2e` and `member-operator`, so you need to run e2e tests against your current code located in `../member-operator` directory:* -*** run `make test-e2e-member-local` -** *You are working in both repos `toolchain-e2e` and `host-operator`, so you need to run e2e tests against your current code located in `../host-operator` directory:* -*** run `make test-e2e-host-local` -** *You are working in all three repos `toolchain-e2e`, `host-operator` and `member-operator`, so you need to run e2e tests against your current code located in both directories `../host-operator` and `../member-operator`:* -*** run `make test-e2e-local` - -* *Creating PRs:* -** *Your PR doesn't need any changes in https://github.com/codeready-toolchain/host-operator[host-operator] repo nor https://github.com/codeready-toolchain/member-operator[member-operator] repo:* -*** 1. check the name of a branch you are going to create a PR for -*** 2. make sure that your forks of both repos (https://github.com/codeready-toolchain/host-operator[host-operator] and https://github.com/codeready-toolchain/member-operator[member-operator]) don't contain a branch with the same name -*** 3. create a PR -** *Your PR requires changes in https://github.com/codeready-toolchain/host-operator[host-operator] repo but not in https://github.com/codeready-toolchain/member-operator[member-operator] repo:* -*** 1. check the name of a branch you are going to create a PR for -*** 2. create a branch with the same name within your fork of https://github.com/codeready-toolchain/host-operator[host-operator] repo and put all necessary changes there -*** 3. make sure that your fork of https://github.com/codeready-toolchain/member-operator[member-operator] repo doesn't contain a branch with the same name -*** 4. push all changes into both forks of the repositories https://github.com/codeready-toolchain/toolchain-e2e[toolchain-e2e] and https://github.com/codeready-toolchain/host-operator[host-operator] -*** 5. create a PR for https://github.com/codeready-toolchain/toolchain-e2e[toolchain-e2e] -*** 6. create a PR for https://github.com/codeready-toolchain/host-operator[host-operator] -** *Your PR requires changes in https://github.com/codeready-toolchain/member-operator[member-operator] repo but not in https://github.com/codeready-toolchain/host-operator[host-operator] repo:* -*** See the previous case and just swap member-operator and host-operator. -** *Your PR requires changes in both repos https://github.com/codeready-toolchain/host-operator[host-operator] and https://github.com/codeready-toolchain/member-operator[member-operator]:* -*** This is prohibited and will result in an error like `ERROR WHILE TRYING TO PAIR PRs` in the CI build. See the reasoning behind this in the <> section. - -== Deploying End-to-End Resources Without Running Tests - -All e2e resources (host operator, member operator, registration-service, CRDs, etc) can be deployed without running tests: - -* `make dev-deploy-e2e-local` - deploys the same resources as `make test-e2e-local` in dev environment but doesn't run tests. - -* `make dev-deploy-e2e` - deploys the same resources as `make test-e2e` in dev environment but doesn't run tests. - -* `make deploy-single-member-e2e-latest` - deploys the same resources (using the latest and greatest images of Toolchain operators) as `make test-e2e` but with only one member and doesn't run tests. - -NOTE: By default these targets deploy resources to `toolchain-host-operator` and `toolchain-member-operator` namespaces. - -NOTE: If running in CodeReady Containers `eval $(crc oc-env)` is required. - -NOTE: By default, `SECOND_MEMBER_MODE` is set to false. - -== How to Test Mailgun/Twilio Notifications in a Dev Environment -* Get a cluster and setup the following env vars -** `export QUAY_NAMESPACE=` -** `export KUBECONFIG=` -* Run `podman login quay.io` -* Create https://github.com/codeready-toolchain/toolchain-infra/tree/master/config/oauth[IdP] -* If you need to change any of the default configuration, modify the ToolchainConfig in https://github.com/codeready-toolchain/toolchain-e2e/blob/master/deploy/host-operator/dev/toolchainconfig.yaml[deploy/host-operator/dev/toolchainconfig.yaml] -* To set working notification/verification secrets, modify them in https://github.com/codeready-toolchain/toolchain-e2e/blob/master/deploy/host-operator/dev/secrets.yaml[deploy/host-operator/dev/secrets.yaml] -* Run `make dev-deploy-e2e-local` -* Go to the registration-service link and sign in -* Click on the `Get Started With CodeReady Toolchain` button -* Approve your usersignup found on the `-host-operator` namespace diff --git a/README.md b/README.md new file mode 100644 index 000000000..a7935cb83 --- /dev/null +++ b/README.md @@ -0,0 +1,176 @@ +# CodeReady Toolchain E2E Tests + +[![Go Report Card](https://goreportcard.com/badge/github.com/codeready-toolchain/toolchain-e2e)](https://goreportcard.com/report/github.com/codeready-toolchain/toolchain-e2e) +[![GoDoc](https://godoc.org/github.com/codeready-toolchain/toolchain-e2e?status.png)](https://godoc.org/github.com/codeready-toolchain/toolchain-e2e) + +This repo contains e2e tests for [host](https://github.com/codeready-toolchain/host-operator) and [member](https://github.com/codeready-toolchain/member-operator) operators of CodeReady Toolchain. + +## Build + +Requires Go version 1.26.x (1.26.5 or higher) - download for your development environment [here](https://golang.org/dl/). + +This repository uses [Go modules](https://github.com/golang/go/wiki/Modules). + +## Step by step guide - running in CodeReady Containers + +Refer to [this guide](openshift_local.md) for detailed instructions on running the e2e tests in a local CodeReady Containers cluster. + +## End-to-End Tests + +The e2e tests are executed against host and member operators running in OpenShift. The operators are built from the [host-operator](https://github.com/codeready-toolchain/host-operator) and [member-operator](https://github.com/codeready-toolchain/member-operator) repositories. + +Since the changes in the e2e repo sometimes require changes in some of the operator repositories at the same time, the logic that executes the e2e tests supports a feature of pairing the e2e PR with **one** other PR based on branch names. +Before the e2e tests are executed in openshift-ci, the logic automatically tries to pair a PR opened for this (toolchain-e2e) repository with a branch of the same name that potentially could exist in any of the developer's fork of the operator repositories. + +For example, if a developer with GH account `cooljohn` opens a PR (for toolchain-e2e repo) from a branch `fix-reconcile`, then the logic checks if there is a branch `fix-reconcile` also in the `cooljohn/host-operator` and `cooljohn/member-operator` forks. +Let's say that `cooljohn/host-operator` contains such a branch but `cooljohn/member-operator` doesn't, then the logic: + +1. clones latest changes of both repos [codeready-toolchain/host-operator](https://github.com/codeready-toolchain/host-operator) and [codeready-toolchain/member-operator](https://github.com/codeready-toolchain/member-operator) +2. fetches the `fix-reconcile` branch from `cooljohn/host-operator` fork +3. merges `master` branch with the changes from `fix-reconcile` branch inside of `host-operator` repo +4. builds images from the merge branch of `host-operator` repo and from `master` branch of `member-operator` repo & deploys them to OpenShift +5. runs e2e tests taken from the opened PR + +It would work analogically also for the case when none of the repositories contain the branch name. However, if **both** repositories contain the branch name then it will result in an error. +This is by design because OLM does not ensure that both the host-operator and member-operator will be updated at the same time in production. Prohibiting PR pairing with more than one repo helps ensure they do not depend on one another and can be updated separately. + +If you still don't know what to do with e2e tests in some use-cases, go to the [What To Do](#what-to-do) section where all use-cases are covered. + +> **Note:** You can also pair a toolchain-e2e PR with a PR from the [ksctl](https://github.com/kubesaw/ksctl) repository, but this is one-directional only (e2e tests are not yet executed in the `ksctl` repo). To rely on a locally installed version of the `ksctl` binary, set `USE_INSTALLED_KSCTL=true`. If you want to build and use the `ksctl` binary from your local `kubesaw/ksctl` repository, set the `KSCTL_REPO_PATH` variable to point to your `kubesaw/ksctl` folder. + +### Running Locally + +**Prerequisites:** + +* Install the [required tools](required_tools.md). +* Configure [your quay account for dev deployment](quay.md). + +#### Running in a Development Environment + +See the procedure to install the Dev Sandbox in a development environment [here](https://kubesaw.github.io/contributing/). + +### Running End-to-End Tests + +1. [Configure your quay account for dev deployment](quay.md) + +Although the e2e tests are in the separated repository than the actual operators are, it's still possible to run them against the current code that is at HEAD of the operator repositories. +There are multiple Makefile targets that will execute the e2e tests, they just differ in where the operators' code is taken from: + +* `make test-e2e` - this target clones the latest changes from both repos [host-operator](https://github.com/codeready-toolchain/host-operator) and [member-operator](https://github.com/codeready-toolchain/member-operator), builds images from the master, deploys to OpenShift and runs e2e tests against them. +* `make test-e2e-local` - this target doesn't clone anything, but it builds operator images from the directories `../host-operator` and `../member-operator`. These images deploys to OpenShift and runs e2e tests against them. +* `make test-e2e-member-local` - this target clones only the [host-operator](https://github.com/codeready-toolchain/host-operator) repo and builds an image from it. For member-operator, it builds the image from `../member-operator` directory. These images deploys to OpenShift and runs e2e tests against them. +* `make test-e2e-host-local` - this target clones only the [member-operator](https://github.com/codeready-toolchain/member-operator) repo and builds an image from it. For host-operator, it builds the image from `../host-operator` directory. These images deploys to OpenShift and runs e2e tests against them. + +The e2e tests will take care of creating all needed namespaces with random names (or see below for enforcing some specific namespace names). +It will also create all required CRDs, role and role bindings for the service accounts, build the container images for both operators and push them to the OpenShift container registry. Finally, it will deploy the operators and run the tests using the operator-sdk. + +> **Note:** You can override the default namespace names where the end-to-end tests are going to be executed - eg.: `make test-e2e HOST_NS=my-host MEMBER_NS=my-member`. + +> **Note:** You can disable SSL/TLS certificate verification in tests setting the `DISABLE_KUBE_CLIENT_TLS_VERIFY` variable to `true` - eg.: `make test-e2e DISABLE_KUBE_CLIENT_TLS_VERIFY=true`. This flag helps when you test in clusters using Self-Signed Certificates. + +> **Note:** You can specify a regular expression to selectively run particular test cases by setting the `TESTS_RUN_FILTER_REGEXP` variable. eg.: `make test-e2e TESTS_RUN_FILTER_REGEXP="TestSetupMigration"`. For more information see the [go test -run documentation](https://pkg.go.dev/cmd/go#hdr-Testing_flags). + +> **Note:** You should not override `SECOND_MEMBER_MODE` in test-e2e, since the e2e tests require a second member operator. + +### Running/Debugging e2e tests from your IDE + +In order to run/debug tests from your IDE you'll need to export some required env variables, those will be used by the test framework to interact with the operator namespaces and the other toolchain resources in you cluster. +Following snippet of code should be TEMPORARILY added at the top of the test you want to run/debug from your IDE: + +```go +os.Setenv("MEMBER_NS","toolchain-member-18161051") +// `SECOND_MEMBER_MODE` should be set to true, since the e2e tests require a second member operator. +os.Setenv("SECOND_MEMBER_MODE","true") +os.Setenv("MEMBER_NS_2","toolchain-member2-18161051") +os.Setenv("HOST_NS","toolchain-host-18161051") +os.Setenv("REGISTRATION_SERVICE_NS","toolchain-host-18161051") +os.Setenv("KUBECONFIG", "~/aws-cluster-test/my-devsandbox/auth/kubeconfig") +``` + +Example of test case code containing the debugging env variables: + +```go +package parallel + +import ( + "context" + "os" + "testing" +) + +func TestCreateSpaceRequest(t *testing.T) { + os.Setenv("MEMBER_NS","toolchain-member-18161051") + os.Setenv("SECOND_MEMBER_MODE","true") + os.Setenv("MEMBER_NS_2","toolchain-member2-18161051") + os.Setenv("HOST_NS","toolchain-host-18161051") + os.Setenv("REGISTRATION_SERVICE_NS","toolchain-host-18161051") + os.Setenv("KUBECONFIG", "~/aws-cluster-test/my-devsandbox/auth/kubeconfig") + // some more code here ... + + t.Run("create space request", func(t *testing.T) { + // test case implementation here ... +.... +``` + +> **Note:** Replace the values with the ones from your dev/test environment and REMEMBER TO REMOVE THE SNIPPET BEFORE COMMITTING THE CODE OR OPENING A PR IN GH :) + +#### What To Do + +If you are still confused by the different e2e/operator location, execution and branch pairing, see the following cases and needed steps: + +* **Working locally:** + * **Need to verify changes in e2e tests against the latest version of both operators:** + * run `make test-e2e` + * **You are working in both repos `toolchain-e2e` and `member-operator`, so you need to run e2e tests against your current code located in `../member-operator` directory:** + * run `make test-e2e-member-local` + * **You are working in both repos `toolchain-e2e` and `host-operator`, so you need to run e2e tests against your current code located in `../host-operator` directory:** + * run `make test-e2e-host-local` + * **You are working in all three repos `toolchain-e2e`, `host-operator` and `member-operator`, so you need to run e2e tests against your current code located in both directories `../host-operator` and `../member-operator`:** + * run `make test-e2e-local` + +* **Creating PRs:** + * **Your PR doesn't need any changes in [host-operator](https://github.com/codeready-toolchain/host-operator) repo nor [member-operator](https://github.com/codeready-toolchain/member-operator) repo:** + 1. check the name of a branch you are going to create a PR for + 2. make sure that your forks of both repos ([host-operator](https://github.com/codeready-toolchain/host-operator) and [member-operator](https://github.com/codeready-toolchain/member-operator)) don't contain a branch with the same name + 3. create a PR + * **Your PR requires changes in [host-operator](https://github.com/codeready-toolchain/host-operator) repo but not in [member-operator](https://github.com/codeready-toolchain/member-operator) repo:** + 1. check the name of a branch you are going to create a PR for + 2. create a branch with the same name within your fork of [host-operator](https://github.com/codeready-toolchain/host-operator) repo and put all necessary changes there + 3. make sure that your fork of [member-operator](https://github.com/codeready-toolchain/member-operator) repo doesn't contain a branch with the same name + 4. push all changes into both forks of the repositories [toolchain-e2e](https://github.com/codeready-toolchain/toolchain-e2e) and [host-operator](https://github.com/codeready-toolchain/host-operator) + 5. create a PR for [toolchain-e2e](https://github.com/codeready-toolchain/toolchain-e2e) + 6. create a PR for [host-operator](https://github.com/codeready-toolchain/host-operator) + * **Your PR requires changes in [member-operator](https://github.com/codeready-toolchain/member-operator) repo but not in [host-operator](https://github.com/codeready-toolchain/host-operator) repo:** + * See the previous case and just swap member-operator and host-operator. + * **Your PR requires changes in both repos [host-operator](https://github.com/codeready-toolchain/host-operator) and [member-operator](https://github.com/codeready-toolchain/member-operator):** + * This is prohibited and will result in an error like `ERROR WHILE TRYING TO PAIR PRs` in the CI build. See the reasoning behind this in the [End-to-End Tests](#end-to-end-tests) section. + +## Deploying End-to-End Resources Without Running Tests + +All e2e resources (host operator, member operator, registration-service, CRDs, etc) can be deployed without running tests: + +* `make dev-deploy-e2e-local` - deploys the same resources as `make test-e2e-local` in dev environment but doesn't run tests. + +* `make dev-deploy-e2e` - deploys the same resources as `make test-e2e` in dev environment but doesn't run tests. + +* `make deploy-single-member-e2e-latest` - deploys the same resources (using the latest and greatest images of Toolchain operators) as `make test-e2e` but with only one member and doesn't run tests. + +> **Note:** By default these targets deploy resources to `toolchain-host-operator` and `toolchain-member-operator` namespaces. + +> **Note:** If running in CodeReady Containers `eval $(crc oc-env)` is required. + +> **Note:** By default, `SECOND_MEMBER_MODE` is set to false. + +## How to Test Mailgun/Twilio Notifications in a Dev Environment + +* Get a cluster and setup the following env vars + * `export QUAY_NAMESPACE=` + * `export KUBECONFIG=` +* Run `podman login quay.io` +* Create [IdP](https://github.com/codeready-toolchain/toolchain-infra/tree/master/config/oauth) +* If you need to change any of the default configuration, modify the ToolchainConfig in [deploy/host-operator/dev/toolchainconfig.yaml](https://github.com/codeready-toolchain/toolchain-e2e/blob/master/deploy/host-operator/dev/toolchainconfig.yaml) +* To set working notification/verification secrets, modify them in [deploy/host-operator/dev/secrets.yaml](https://github.com/codeready-toolchain/toolchain-e2e/blob/master/deploy/host-operator/dev/secrets.yaml) +* Run `make dev-deploy-e2e-local` +* Go to the registration-service link and sign in +* Click on the `Get Started With CodeReady Toolchain` button +* Approve your usersignup found on the `-host-operator` namespace diff --git a/go.mod b/go.mod index 2c2be462c..836fdaf0f 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,12 @@ module github.com/codeready-toolchain/toolchain-e2e +go 1.26.0 + +toolchain go1.26.5 + require ( - github.com/codeready-toolchain/api v0.0.0-20260731065234-734640c901d2 - github.com/codeready-toolchain/toolchain-common v0.0.0-20260804130313-0d4161d82454 + github.com/codeready-toolchain/api v0.0.0-20260807111559-e29da2fc346c + github.com/codeready-toolchain/toolchain-common v0.0.0-20260807125728-33faed3f17f9 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/fatih/color v1.18.0 github.com/ghodss/yaml v1.0.0 @@ -108,13 +112,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.3 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.55.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect - golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/term v0.37.0 // indirect - golang.org/x/text v0.31.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/term v0.43.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.8 // indirect @@ -136,8 +140,4 @@ require ( sigs.k8s.io/yaml v1.6.0 // indirect ) -go 1.24.4 - -toolchain go1.24.13 - tool github.com/mxschmitt/playwright-go/cmd/playwright diff --git a/go.sum b/go.sum index 988871e7a..fc69cec5f 100644 --- a/go.sum +++ b/go.sum @@ -18,10 +18,10 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5O github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/codeready-toolchain/api v0.0.0-20260731065234-734640c901d2 h1:i5izQYQWwi0Dbc2Mv/sDOJ5GIMH69O3n0jRwQaAWo9o= -github.com/codeready-toolchain/api v0.0.0-20260731065234-734640c901d2/go.mod h1:PMg6kNHuCGNlu3MOdrCisqGkBpvzB0qS1+E6nrXxPAc= -github.com/codeready-toolchain/toolchain-common v0.0.0-20260804130313-0d4161d82454 h1:D3+dc9mEmLrq0XiG6xXDVJoZ+YChF7eX1OTKZ62ydfU= -github.com/codeready-toolchain/toolchain-common v0.0.0-20260804130313-0d4161d82454/go.mod h1:j4xJuhNtqEuZW7VR6BB5V6h/oDJ0dIwSh5iijUSwwdM= +github.com/codeready-toolchain/api v0.0.0-20260807111559-e29da2fc346c h1:0fyFQlMi89U2ML+BjwEI3CW4MRJMJjQL7+RjrypxRDQ= +github.com/codeready-toolchain/api v0.0.0-20260807111559-e29da2fc346c/go.mod h1:jkXNLbwYNA3rjOHZ6z6m8BsCs467LmveCapPQFfQZNY= +github.com/codeready-toolchain/toolchain-common v0.0.0-20260807125728-33faed3f17f9 h1:VKIQ1mvALQrHEt4aSlZlNcfXcGX2zCOcskiE2G97RVU= +github.com/codeready-toolchain/toolchain-common v0.0.0-20260807125728-33faed3f17f9/go.mod h1:TXPJQ/4BLJUA+B2sRnEmxynLJy/HIzMo42xNMEDIilI= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -271,8 +271,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -287,8 +287,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -296,8 +296,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -314,8 +314,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -323,8 +323,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -332,8 +332,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -342,8 +342,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= -golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/gofmt_exclude b/gofmt_exclude deleted file mode 100644 index fa4d83322..000000000 --- a/gofmt_exclude +++ /dev/null @@ -1 +0,0 @@ -vendor/.* \ No newline at end of file diff --git a/make/format.mk b/make/format.mk deleted file mode 100644 index 037bd0aea..000000000 --- a/make/format.mk +++ /dev/null @@ -1,17 +0,0 @@ -GOFORMAT_FILES := $(shell find . -name '*.go' | grep -vEf ./gofmt_exclude) - -.PHONY: check-go-format -## Verify the formatting defined by 'gofmt' -check-go-format: - $(Q)gofmt -s -l ${GOFORMAT_FILES} 2>&1 \ - | tee $(OUT_DIR)/gofmt-errors \ - | read \ - && echo "ERROR: These files differ from gofmt's style (run 'make format-go-code' to fix this):" \ - && cat $(OUT_DIR)/gofmt-errors \ - && exit 1 \ - || true - -.PHONY: format-go-code -## Formats any go file that does not match formatting defined by gofmt -format-go-code: - $(Q)gofmt -s -l -w ${GOFORMAT_FILES} \ No newline at end of file diff --git a/make/go.mk b/make/go.mk index 00a7e0c29..5fe6207e5 100644 --- a/make/go.mk +++ b/make/go.mk @@ -7,6 +7,14 @@ GO111MODULE?=on export GO111MODULE goarch?=$(shell go env GOARCH) +.PHONY: format-go-code +## Formats any go file that does not match formatting defined by gofmt +format-go-code: +# The + tells find to batch multiple found files into a single gofmt invocation (like xargs), +# which is much faster than the alternative \;, which runs gofmt once per file. Removing it +# would be a syntax error — find -exec requires either + or \; as a terminator. + $(Q)find . -name '*.go' -not -path '*/vendor/*' -not -path '*/.git/*' -exec gofmt -s -l -w {} + + .PHONY: build ## Build e2e test files build: diff --git a/multicluster_setup.adoc b/multicluster_setup.md similarity index 89% rename from multicluster_setup.adoc rename to multicluster_setup.md index 23b682e27..7681c94a6 100644 --- a/multicluster_setup.adoc +++ b/multicluster_setup.md @@ -1,30 +1,28 @@ -= Multi-Cluster Setup (This document is outdated!!!) +# Multi-Cluster Setup (This document is outdated!!!) Prerequisites: -You need to boostrap 2 clusters and make sure you have a `kubeconfig` file for each cluster in a dedicated/separate location. +You need to bootstrap 2 clusters and make sure you have a `kubeconfig` file for each cluster in a dedicated/separate location. Also, make sure that you already ran the `oc --kubeconfig= login` on each cluster. -== Setting up the KUBECONFIG +## Setting up the KUBECONFIG -First, let's setup a single `KUBECONFIG` with multiple contexts. Assuming that `${HOST_CONFIG}` is the path to the `KUBECONFIG` file for your *Host* cluster, and `${MEMBER_CONFIG}` is the path to the `KUBECONFIG` file for your *Member* cluster: +First, let's setup a single `KUBECONFIG` with multiple contexts. Assuming that `${HOST_CONFIG}` is the path to the `KUBECONFIG` file for your **Host** cluster, and `${MEMBER_CONFIG}` is the path to the `KUBECONFIG` file for your **Member** cluster: -[source,bash] ----- +```bash $ oc --kubeconfig ${HOST_CONFIG} config rename-context `oc --kubeconfig ${HOST_CONFIG} config current-context` host-admin $ oc --kubeconfig ${MEMBER_CONFIG} config rename-context `oc --kubeconfig ${MEMBER_CONFIG} config current-context` member-admin # join the 2 files to "generate" a single KUBECONFIG $ export KUBECONFIG=${HOST_CONFIG}:${MEMBER_CONFIG} ----- +``` At this point, we can switch the terminal connection from one cluster to another use the `oc config use-context host-admin` and `oc config use-context member-admin` commands. You can verify by yourself with: -[source,bash] ----- +```bash $ oc config use-context host-admin && oc whoami --show-server Switched to context "host-admin". https://api..devcluster.openshift.com:6443 @@ -32,40 +30,35 @@ https://api..devcluster.openshift.com:6443 $ oc config use-context member-admin && oc whoami --show-server Switched to context "member-admin". https://api..devcluster.openshift.com:6443 - ----- +``` Once both contexts are configured, we can move on with the operator deployments, starting with the Host operator. -[source,bash] ----- - +```bash $ export HOST_NS=toolchain-host-operator $ export REGISTRATION_SERVICE_NS=${HOST_NS} # deploy in the specific `toolchain-host-operator` namespace $ oc config use-context host-admin && HOST_NS=${HOST_NS} make get-host-and-reg-service deploy-host ----- +``` This will clone the remote https://github.com/codeready-toolchain/host-operator.git and https://github.com/codeready-toolchain/registration-service.git repositories in `/tmp/codeready-toolchain/` and build the operator and service from these temporary directories, then deploy the Host operator on the Host cluster. After a minute (or more), both `host-operator` and `registration-service` pods should be available: -[source,bash] ----- +```bash $ oc get pods -n ${HOST_NS} NAME READY STATUS RESTARTS AGE host-operator-...-... 1/1 Running 0 ... registration-service-...-... 1/1 Running 0 ... registration-service-...-... 1/1 Running 0 ... ----- +``` -== Member Cluster Setup +## Member Cluster Setup The same logic as above is applied to deploy the Member operator on the Member cluster: -[source,bash] ----- +```bash $ export MEMBER_NS=toolchain-member-operator # on the member cluster, in the `toolchain-member-operator` namespace @@ -75,16 +68,15 @@ $ oc config use-context member-admin && MEMBER_NS_TO_DEPLOY=${MEMBER_NS} make ge $ oc get pods -n ${MEMBER_NS} NAME READY STATUS RESTARTS AGE member-operator-...-... 1/1 Running 0 ... ----- +``` -== Configuring the Operators +## Configuring the Operators Once Host and Member operators have been deployed on their respective cluster, they need to be configured to communicate with each other. At this stage, each operator has its own status resource, which is in a `Ready=False` status, because the host-member connection is not set yet: -[source,bash] ----- +```bash # on the host cluster $ oc config use-context host-admin && oc get toolchainstatus -n ${HOST_NS} NAME READY LAST UPDATED @@ -94,10 +86,9 @@ toolchain-status False ... $ oc config use-context member-admin && oc get memberstatus -n ${MEMBER_NS} NAME READY LAST UPDATED toolchain-member-status False ... ----- +``` -[source,bash] ----- +```bash # create/configure the ToolchainCluster resources on host and member clusters $ ksctl adm register-member --host-ns="${HOST_NS}" --member-ns="${MEMBER_NS}" --host-kubeconfig="${HOME}/.kube/host-config" --member-kubeconfig="${HOME}/.kube/member-config" @@ -121,17 +112,16 @@ $ oc config use-context member-admin && oc get toolchainclusters -n ${MEMBER_NS} Switched to context "member-admin". NAME AGE READY host-... ... True ----- +``` At this point, Host and Member clusters are ready to use. -[source,bash] ----- +```bash $ oc config use-context host-admin && oc project ${HOST_NS} Switched to context "host-admin". Now using project "toolchain-host-operator" on server ... $ HOST_NS=${HOST_NS} MEMBER_NS=${MEMBER_NS} operator-sdk test local ./test/e2e --no-setup --operator-namespace toolchain-e2e --verbose --go-test-flags "-test.timeout=30m -test.failfast -run TestE2E -v" ----- +``` Have fun! diff --git a/openshift_local.adoc b/openshift_local.md similarity index 83% rename from openshift_local.adoc rename to openshift_local.md index 3987a8e7e..39832f3da 100644 --- a/openshift_local.adoc +++ b/openshift_local.md @@ -1,9 +1,7 @@ -:imagesdir: doc/images +# Setting up OpenShift Local (formerly CRC, CodeReady Containers) — Step-by-step guide -== Setting up OpenShift Local (formerly CRC, CodeReady Containers) — Step-by-step guide - -IMPORTANT: OpenShift Local includes an embedded system bundle that contains certificates which expire 30 days -after the release. Because of this it is very important to always run the latest release of OpenShift Local. +> **Important:** OpenShift Local includes an embedded system bundle that contains certificates which expire 30 days +> after the release. Because of this it is very important to always run the latest release of OpenShift Local. OpenShift Local is a distribution of OpenShift designed to be run on a development PC, and while some features have been disabled by default it is still quite demanding in terms of system resources, and for this reason it is @@ -12,50 +10,47 @@ recommended that it be installed on a machine that has at least 32GB of memory. This guide will walk through the steps of downloading and installing OpenShift Local, and running the e2e tests against local CodeReady Toolchain `host-operator` and `member-operator` repositories. -=== Install the required tools +## Install the required tools -Please check the xref:required_tools.adoc[Required Tools] page and install those tools and utilities before proceeding, +Please check the [Required Tools](required_tools.md) page and install those tools and utilities before proceeding, as otherwise you will experience test failures and having to restart the tests from the beginning. -=== Download and install OpenShift Local -Download OpenShift Local from https://developers.redhat.com/products/openshift-local/overview[developers.redhat.com]. +## Download and install OpenShift Local + +Download OpenShift Local from [developers.redhat.com](https://developers.redhat.com/products/openshift-local/overview). You will need to log in using your Red Hat SSO account, after which you may click on the `Install OpenShift on your laptop` button which will take you to the download page for OpenShift Local. From here, select your OS before clicking `Download OpenShift Local`. You will also need to download your pull secret, and keep that in a safe place. -image::openshift_local_download.png[align="center"] +![OpenShift Local download](doc/images/openshift_local_download.png) Extract the downloaded file into a directory of your choice: -[source,bash] ----- +```bash tar -xvf crc-linux-amd64.tar.xz ----- +``` Give execution permissions to the binary and move it to a directory on your path or `/usr/local/bin`: -[source,bash] ----- +```bash chmod u+x crc sudo mv crc /usr/local/bin ----- +``` -=== Set up the cluster and enable or tweak the cluster's settings +## Set up the cluster and enable or tweak the cluster's settings You need to set up the OpenShift Local cluster — the daemons, configurations and basic settings for it to be able to run — by running the command below. You only need to do it the first time you're setting up the cluster or after running `crc cleanup`: -[source,bash] ----- +```bash crc setup ----- +``` Also, in order to run the tests seamlessly and without any problems, there is a set of settings you are advised to change: -[source,bash] ----- +```bash # Cluster monitoring is required for the tests to pass crc config set enable-cluster-monitoring true @@ -64,33 +59,32 @@ crc config set enable-cluster-monitoring true crc config set cpus 6 crc config set disk-size 50 crc config set memory 14500 # in MB. You can also use 20000 for 20GB, to be safer. ----- +``` Now you can go ahead and start the cluster. The first time you will need to provide the pull secret you were presented with in the OpenShift Local download page — if you didn't grab it you can come back and copy it. Run the following command to start the cluster: -[source,bash] ----- +```bash crc start ----- +``` While your local OpenShift cluster boots, you can go ahead and prepare the Quay repositories to be able to run the tests. -=== Final configurations -==== Creating the Quay repositories and making them public +## Final configurations + +### Creating the Quay repositories and making them public -Please follow the steps in the xref:quay.adoc["Configure your Quay account for dev deployment"] document to set up the +Please follow the steps in the [Configure your Quay account for dev deployment](quay.md) document to set up the Quay repositories, and then come back to this guide. -==== Logging in to your cluster and Quay +### Logging in to your cluster and Quay After some time has passed, the local OpenShift cluster might be ready to work with. The terminal should show an output similar to the following one: -[source,text] ----- +```text INFO Using bundle path /home/${USER}/.crc/cache/crc_libvirt_4.19.8_amd64.crcbundle INFO Checking if running as non-root INFO Checking if running inside WSL2 @@ -175,52 +169,47 @@ Log in as user: Use the 'oc' command line interface: $ eval $(crc oc-env) $ oc login -u developer https://api.crc.testing:6443 ----- +``` Add the `oc` executable to your current path by running the following command: -[source,bash] ----- +```bash eval $(crc oc-env) ----- +``` -Now, log in as the *kubeadmin* user, since you need the required privileges to manage namespaces, install operators, +Now, log in as the **kubeadmin** user, since you need the required privileges to manage namespaces, install operators, clean up resources and what not. Your login command should look something similar to this: -[source,bash] ----- +```bash oc login -u kubeadmin -p ${KUBEADMIN_PASSWORD} https://api.crc.testing:6443 ----- +``` Also, make sure you're logged in to Quay with Podman: -[source,bash] ----- +```bash podman login quay.io ----- +``` -==== Running the tests +### Running the tests -Please follow the steps in xref:README.adoc#_running_end_to_end_tests["README § Running End-to-End tests"] to set up +Please follow the steps in [README - Running End-to-End Tests](README.md#running-end-to-end-tests) to set up the local operator repositories — if any — and run the tests. -=== Cleaning up +## Cleaning up After a run, regardless of whether it was successful or not, you can — and it is recommended to — run the following target to clean up the resources in the local OpenShift cluster: -[source,bash] ----- +```bash make clean-e2e-resources ----- +``` If for some reason the cleaning up of the resources gets stuck, you can run the following target before running the "clean" target again to remove the finalizers that prevent the cleanup: -[source,bash] ----- +```bash make force-remove-finalizers-from-e2e-resources # Rerun the cleanup again. make clean-e2e-resources ----- +``` diff --git a/quay.adoc b/quay.adoc deleted file mode 100644 index 794c414a1..000000000 --- a/quay.adoc +++ /dev/null @@ -1,40 +0,0 @@ -== Configure your Quay account for dev deployment - -There is a set of images that are built and pushed to quay repositories while deploying local versions of Toolchain -(Sandbox) operators to OpenShift cluster. Please make sure that the repositories exist in your Quay.io account. - -=== Repositories -. Register for a quay.io account if you don't have one and log in to the account. -. Go to the repository section, or click on the following link: https://quay.io/repository. -. Click on the "Create new repository" button. -. Select your personal namespace if it is not already selected for you, give the repository an appropriate name, and -choose "Public" as the repository's visibility. -. Click on "Create". - -Also, make sure that: - -. You have set the `QUAY_NAMESPACE` environment variable so that any commands or tests you run use your personal Quay -repositories: `export QUAY_NAMESPACE=` -. You are logged in to your Quay.io account in Podman too with `podman login quay.io` for the same previous reason. -. You end up with the following *public* repositories in Quay: -.. https://quay.io/repository//host-operator -.. https://quay.io/repository//host-operator-bundle -.. https://quay.io/repository//host-operator-index -.. https://quay.io/repository//member-operator -.. https://quay.io/repository//member-operator-webhook -.. https://quay.io/repository//member-operator-bundle -.. https://quay.io/repository//member-operator-index -.. https://quay.io/repository//registration-service - - -=== Public visibility -All aforementioned repositories has to be public, so make sure that the visibility is set to `public` for all of them: - -* https://quay.io/repository//host-operator?tab=settings -* https://quay.io/repository//host-operator-bundle?tab=settings -* https://quay.io/repository//host-operator-index?tab=settings -* https://quay.io/repository//member-operator?tab=settings -* https://quay.io/repository//member-operator-webhook?tab=settings -* https://quay.io/repository//member-operator-bundle?tab=settings -* https://quay.io/repository//member-operator-index?tab=settings -* https://quay.io/repository//registration-service?tab=settings diff --git a/quay.md b/quay.md new file mode 100644 index 000000000..f061c1626 --- /dev/null +++ b/quay.md @@ -0,0 +1,41 @@ +# Configure your Quay account for dev deployment + +There is a set of images that are built and pushed to quay repositories while deploying local versions of Toolchain +(Sandbox) operators to OpenShift cluster. Please make sure that the repositories exist in your Quay.io account. + +## Repositories + +1. Register for a quay.io account if you don't have one and log in to the account. +2. Go to the repository section, or click on the following link: https://quay.io/repository. +3. Click on the "Create new repository" button. +4. Select your personal namespace if it is not already selected for you, give the repository an appropriate name, and + choose "Public" as the repository's visibility. +5. Click on "Create". + +Also, make sure that: + +1. You have set the `QUAY_NAMESPACE` environment variable so that any commands or tests you run use your personal Quay + repositories: `export QUAY_NAMESPACE=` +2. You are logged in to your Quay.io account in Podman too with `podman login quay.io` for the same previous reason. +3. You end up with the following **public** repositories in Quay: + * `https://quay.io/repository//host-operator` + * `https://quay.io/repository//host-operator-bundle` + * `https://quay.io/repository//host-operator-index` + * `https://quay.io/repository//member-operator` + * `https://quay.io/repository//member-operator-webhook` + * `https://quay.io/repository//member-operator-bundle` + * `https://quay.io/repository//member-operator-index` + * `https://quay.io/repository//registration-service` + +## Public visibility + +All aforementioned repositories have to be public, so make sure that the visibility is set to `public` for all of them: + +* `https://quay.io/repository//host-operator?tab=settings` +* `https://quay.io/repository//host-operator-bundle?tab=settings` +* `https://quay.io/repository//host-operator-index?tab=settings` +* `https://quay.io/repository//member-operator?tab=settings` +* `https://quay.io/repository//member-operator-webhook?tab=settings` +* `https://quay.io/repository//member-operator-bundle?tab=settings` +* `https://quay.io/repository//member-operator-index?tab=settings` +* `https://quay.io/repository//registration-service?tab=settings` diff --git a/required_tools.adoc b/required_tools.adoc deleted file mode 100644 index 27306598a..000000000 --- a/required_tools.adoc +++ /dev/null @@ -1,11 +0,0 @@ -== Required Pre-installed Tools -* go 1.24.x (1.24.13 or higher) -* git -* operator-sdk 1.42.0 + -NOTE: Follow the installation instructions https://sdk.operatorframework.io/docs/installation/#install-from-github-release[here]. Make sure that the download URL (specified by the `OPERATOR_SDK_DL_URL` environment variable) is set to the correct version. -* sed -* yamllint -* jq -* podman -* opm v1.59.0 + -NOTE: To download the Operator Registry tool use either https://github.com/operator-framework/operator-registry/releases or https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/. The version should correspond with the OpenShift version you are running. To confirm that the Operator Registry tool is installed correctly: `$ opm version` diff --git a/required_tools.md b/required_tools.md new file mode 100644 index 000000000..a1237564d --- /dev/null +++ b/required_tools.md @@ -0,0 +1,15 @@ +# Required Pre-installed Tools + +* go 1.26.x (1.26.5 or higher) +* git +* operator-sdk 1.42.0+ + +> **Note:** Follow the installation instructions [here](https://sdk.operatorframework.io/docs/installation/#install-from-github-release). Make sure that the download URL (specified by the `OPERATOR_SDK_DL_URL` environment variable) is set to the correct version. + +* sed +* yamllint +* jq +* podman +* opm v1.59.0+ + +> **Note:** To download the Operator Registry tool use either https://github.com/operator-framework/operator-registry/releases or https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/. The version should correspond with the OpenShift version you are running. To confirm that the Operator Registry tool is installed correctly: `$ opm version` diff --git a/setup/README.adoc b/setup/README.adoc deleted file mode 100644 index 2a76a98a0..000000000 --- a/setup/README.adoc +++ /dev/null @@ -1,163 +0,0 @@ -= Dev Sandbox Performance Testing - -This document describes how to use the setup tool to set up a Dev Sandbox environment that is similar to production and is used as part of an onboarding process for new operators. - -== Prereqs - -. Ensure your go version matches the specified version in https://github.com/codeready-toolchain/toolchain-e2e/blob/master/go.mod -. Provision the *latest available* GA version of *OCP 4.21.x* on AWS with sufficient resources: 3 `m5.8xlarge` master nodes and 3 `m5.2xlarge` worker nodes. -+ -The latest version of openshift-install can be downloaded from https://mirror.openshift.com/pub/openshift-v4/clients/ocp/ -+ -For example, if using the openshift-install tool to install OCP on AWS you can create an install-config.yaml file that has the following configuration: -+ ----- -apiVersion: v1 -baseDomain: devcluster.openshift.com -controlPlane: - hyperthreading: Enabled - name: master - platform: - aws: - type: "m5.8xlarge" - replicas: 3 -compute: -- hyperthreading: Enabled - name: worker - platform: - aws: - type: "m5.2xlarge" - replicas: 3 -metadata: - name: sandbox-test -networking: - clusterNetwork: - - cidr: 10.128.0.0/14 - hostPrefix: 23 - machineNetwork: - - cidr: 10.0.0.0/16 - networkType: OVNKubernetes - serviceNetwork: - - 172.30.0.0/16 -platform: - aws: - region: us-east-1 -publish: External ----- - -. Log in to the cluster using `oc login --token= --server=`, this is required for the tool to interact with the cluster and gather metrics during the run. -.. Login to the cluster's OpenShift console using `kubeadmin` user -.. Click the user dropdown menu on the top right of the screen -.. Select "Copy login command" -.. Copy the oc login command with token and run the command in your terminal before proceeding running the setup tool -.. Note: You may need to include `--insecure-skip-tls-verify=true` when running the oc login command. - -. Install the https://github.com/codeready-toolchain/toolchain-e2e/blob/master/required_tools.adoc[required tools]. - -. Install the onboarding operator(s) manually. Other operators that have already been onboarded will be installed automatically by the tool. - -. Create an OpenShift template file (onboarding.yaml) that defines resources for testing the performance of your onboarding operator and any other resources that users typically create when using your operator. A Dev Sandbox template is provided with a default set of resources to help mimic a Dev Sandbox production environment https://raw.githubusercontent.com/codeready-toolchain/toolchain-e2e/master/setup/resources/user-workloads.yaml[user-workloads.yaml]. -+ -The setup tool will automatically create resources on behalf of the users in their `stage` namespaces. The resources are defined in template files and fed to the tool using the `--template` parameter. -+ -Note #1: All resources will be created in the user's `-stage` namespace regardless of whether resources in the template have a namespace set. -Note #2: Only resources that a user has permissions to create will be successfully created, these are typically namespace-scoped resources limited to only the user's namespaces. If the tool fails to create any resources an error will occur. If these resources are required by the onboarding operator then this should be brought to the attention of the Dev Sandbox team. - -== Dev Sandbox Operators Setup - -. Clone this repository -+ -``` -git clone git@github.com:codeready-toolchain/toolchain-e2e.git -``` -. Run the following to install the Dev Sandbox operators -+ -``` -make dev-deploy-latest -``` -. Run the following command and ensure the Ready status is `True` -+ -``` -oc get toolchainstatus -n toolchain-host-operator -``` -+ -``` -NAME MURS READY LAST UPDATED -toolchain-status 0 True 2021-03-24T22:39:36Z -``` - -== Provisioning Test Users And Capturing Metrics - -*IMPORTANT: Performance results may be skewed when a fresh cluster is not used. Results for performance comparison and operator onboarding purposes should be captured using a fresh cluster.* - -. Log in to the cluster using the `kubeadmin` user via `oc login --token= --server=` if you haven't already done so. - -. Run the setup with a single user to verify all the operators can be installed and capture metrics after installing all operators but before provisioning the 2000 users. -+ -``` -go run setup/main.go --users 1 --default 1 --custom 0 --username setup -``` -+ -After the command completes it will print performance metrics that can be used for comparison against the baseline metrics. The results are saved to a .csv file to make it easier to copy the results into the spreadsheet. -+ -Add the results to the Onboarding Performance Checklist spreadsheet in the `Onboarding Operator 1 user` column. -+ -. Populate the cluster with 2000 users along with default and custom resources for each user. -+ -Run the following command to create 2000 users -+ -``` -go run setup/main.go --template= --users 2000 --default 2000 --custom 2000 --username cupcake --workloads namespace:deploymentName -``` -+ -Note 1: You do not need to add the default template (https://raw.githubusercontent.com/codeready-toolchain/toolchain-e2e/master/setup/resources/user-workloads.yaml[setup/resources/user-workloads.yaml]), it is automatically added when you run the setup. You can control how many users will have the default template applied using the `--default` flag. -+ -Note 2: The `--workloads` flag tells the tool to capture the CPU and memory of a deployment and include the results in the summary upon completion of the setup. Use this for including any deployments related to the onboarding operator. The format must follow `--workloads namespace:name` -+ -Note 3: CSV resources are automatically created for each default user as well. An all-namespaces scoped operator will be installed as part of the 'preparing' step. This operator will create a CSV resource in each namespace to mimic the behaviour observed in the production cluster. This operator install step can be skipped with the `--skip-csvgen` flag but should not be skipped without good reason. -+ -Note 4: If your workload is provisioning pods into the user's namespaces the Sandbox operator will delete the pod after an idle timeout of 15 seconds by default. This idle timeout can be configured by setting the `--idler-timeout` parameter like `--idler-timeout 5m` if you want your pods to remain active for longer. -+ -Use `go run setup/main.go --help` to see the full set of options. + -. Grab some coffee ☕️, populating the cluster with 2000 users usually takes about an hour but can take longer depending on network latency + -Note: If for some reason the provisioning users step does not complete (eg. timeout), note down how many users were created and rerun the command with the remaining number of users to be created and a different username prefix. eg. `go run setup/main.go --template= --username zorro --users --default --custom ` -+ -. After the command completes it will print performance metrics that can be used for comparison against the baseline metrics. -+ -Copy these values to the Onboarding Performance Checklist spreadsheet. Add the results to the `Onboarding Operator 2k users` column. The results are saved to a .csv file to make it easier to copy the results into the spreadsheet. - -=== Evaluate the Cluster and Operator(s) - -Wait until all users have been created in the previous step. With the cluster now fully under load, it's time to evaluate the environment. - -1. Use your operators as a user would and evaluate the performance. -2. Monitor the cluster's performance using the Monitoring view in the OpenShift Console. -3. Monitor the memory usage of operators. There are many more resources created on this cluster than most operators have been tested with so it's important to look for any possible areas of concern. -4. Compare the Results summary to the Baseline metrics provided in the onboarding doc. - -== Clean up - -=== Remove Only Users and Their Namespaces - -``` -make clean-users -``` - -*Note: If rerunning the tool for performance comparison purposes a fresh cluster should be used to maintain accuracy.* - -=== Remove All Sandbox-related Resources -``` -make clean-e2e-resources -``` - -== Baseline Runs (Done by the Sandbox team) - -1. Install operators -``` -go run setup/main.go --users 1 --default 1 --custom 0 --username baseline --testname=baseline -``` - -2. Run setup for 2000 users -``` -go run setup/main.go --users 2000 --default 2000 --custom 0 --username cupcake --testname=run1 -``` diff --git a/setup/README.md b/setup/README.md new file mode 100644 index 000000000..fcebe050a --- /dev/null +++ b/setup/README.md @@ -0,0 +1,171 @@ +# Dev Sandbox Performance Testing + +This document describes how to use the setup tool to set up a Dev Sandbox environment that is similar to production and is used as part of an onboarding process for new operators. + +## Prereqs + +1. Ensure your go version matches the specified version in https://github.com/codeready-toolchain/toolchain-e2e/blob/master/go.mod + +2. Provision the **latest available** GA version of **OCP 4.21.x** on AWS with sufficient resources: 3 `m5.8xlarge` master nodes and 3 `m5.2xlarge` worker nodes. + + The latest version of openshift-install can be downloaded from https://mirror.openshift.com/pub/openshift-v4/clients/ocp/ + + For example, if using the openshift-install tool to install OCP on AWS you can create an install-config.yaml file that has the following configuration: + + ```yaml + apiVersion: v1 + baseDomain: devcluster.openshift.com + controlPlane: + hyperthreading: Enabled + name: master + platform: + aws: + type: "m5.8xlarge" + replicas: 3 + compute: + - hyperthreading: Enabled + name: worker + platform: + aws: + type: "m5.2xlarge" + replicas: 3 + metadata: + name: sandbox-test + networking: + clusterNetwork: + - cidr: 10.128.0.0/14 + hostPrefix: 23 + machineNetwork: + - cidr: 10.0.0.0/16 + networkType: OVNKubernetes + serviceNetwork: + - 172.30.0.0/16 + platform: + aws: + region: us-east-1 + publish: External + ``` + +3. Log in to the cluster using `oc login --token= --server=`, this is required for the tool to interact with the cluster and gather metrics during the run. + 1. Login to the cluster's OpenShift console using `kubeadmin` user + 2. Click the user dropdown menu on the top right of the screen + 3. Select "Copy login command" + 4. Copy the oc login command with token and run the command in your terminal before proceeding running the setup tool + 5. Note: You may need to include `--insecure-skip-tls-verify=true` when running the oc login command. + +4. Install the [required tools](https://github.com/codeready-toolchain/toolchain-e2e/blob/master/required_tools.md). + +5. Install the onboarding operator(s) manually. Other operators that have already been onboarded will be installed automatically by the tool. + +6. Create an OpenShift template file (onboarding.yaml) that defines resources for testing the performance of your onboarding operator and any other resources that users typically create when using your operator. A Dev Sandbox template is provided with a default set of resources to help mimic a Dev Sandbox production environment [user-workloads.yaml](https://raw.githubusercontent.com/codeready-toolchain/toolchain-e2e/master/setup/resources/user-workloads.yaml). + + The setup tool will automatically create resources on behalf of the users in their `stage` namespaces. The resources are defined in template files and fed to the tool using the `--template` parameter. + + Note #1: All resources will be created in the user's `-stage` namespace regardless of whether resources in the template have a namespace set. + Note #2: Only resources that a user has permissions to create will be successfully created, these are typically namespace-scoped resources limited to only the user's namespaces. If the tool fails to create any resources an error will occur. If these resources are required by the onboarding operator then this should be brought to the attention of the Dev Sandbox team. + +## Dev Sandbox Operators Setup + +1. Clone this repository + + ``` + git clone git@github.com:codeready-toolchain/toolchain-e2e.git + ``` + +2. Run the following to install the Dev Sandbox operators + + ``` + make dev-deploy-latest + ``` + +3. Run the following command and ensure the Ready status is `True` + + ``` + oc get toolchainstatus -n toolchain-host-operator + ``` + + ``` + NAME MURS READY LAST UPDATED + toolchain-status 0 True 2021-03-24T22:39:36Z + ``` + +## Provisioning Test Users And Capturing Metrics + +**IMPORTANT: Performance results may be skewed when a fresh cluster is not used. Results for performance comparison and operator onboarding purposes should be captured using a fresh cluster.** + +1. Log in to the cluster using the `kubeadmin` user via `oc login --token= --server=` if you haven't already done so. + +2. Run the setup with a single user to verify all the operators can be installed and capture metrics after installing all operators but before provisioning the 2000 users. + + ``` + go run setup/main.go --users 1 --default 1 --custom 0 --username setup + ``` + + After the command completes it will print performance metrics that can be used for comparison against the baseline metrics. The results are saved to a .csv file to make it easier to copy the results into the spreadsheet. + + Add the results to the Onboarding Performance Checklist spreadsheet in the `Onboarding Operator 1 user` column. + +3. Populate the cluster with 2000 users along with default and custom resources for each user. + + Run the following command to create 2000 users + + ``` + go run setup/main.go --template= --users 2000 --default 2000 --custom 2000 --username cupcake --workloads namespace:deploymentName + ``` + + Note 1: You do not need to add the default template ([setup/resources/user-workloads.yaml](https://raw.githubusercontent.com/codeready-toolchain/toolchain-e2e/master/setup/resources/user-workloads.yaml)), it is automatically added when you run the setup. You can control how many users will have the default template applied using the `--default` flag. + + Note 2: The `--workloads` flag tells the tool to capture the CPU and memory of a deployment and include the results in the summary upon completion of the setup. Use this for including any deployments related to the onboarding operator. The format must follow `--workloads namespace:name` + + Note 3: CSV resources are automatically created for each default user as well. An all-namespaces scoped operator will be installed as part of the 'preparing' step. This operator will create a CSV resource in each namespace to mimic the behaviour observed in the production cluster. This operator install step can be skipped with the `--skip-csvgen` flag but should not be skipped without good reason. + + Note 4: If your workload is provisioning pods into the user's namespaces the Sandbox operator will delete the pod after an idle timeout of 15 seconds by default. This idle timeout can be configured by setting the `--idler-timeout` parameter like `--idler-timeout 5m` if you want your pods to remain active for longer. + + Use `go run setup/main.go --help` to see the full set of options. + +4. Grab some coffee, populating the cluster with 2000 users usually takes about an hour but can take longer depending on network latency. + + Note: If for some reason the provisioning users step does not complete (eg. timeout), note down how many users were created and rerun the command with the remaining number of users to be created and a different username prefix. eg. `go run setup/main.go --template= --username zorro --users --default --custom ` + +5. After the command completes it will print performance metrics that can be used for comparison against the baseline metrics. + + Copy these values to the Onboarding Performance Checklist spreadsheet. Add the results to the `Onboarding Operator 2k users` column. The results are saved to a .csv file to make it easier to copy the results into the spreadsheet. + +### Evaluate the Cluster and Operator(s) + +Wait until all users have been created in the previous step. With the cluster now fully under load, it's time to evaluate the environment. + +1. Use your operators as a user would and evaluate the performance. +2. Monitor the cluster's performance using the Monitoring view in the OpenShift Console. +3. Monitor the memory usage of operators. There are many more resources created on this cluster than most operators have been tested with so it's important to look for any possible areas of concern. +4. Compare the Results summary to the Baseline metrics provided in the onboarding doc. + +## Clean up + +### Remove Only Users and Their Namespaces + +``` +make clean-users +``` + +**Note: If rerunning the tool for performance comparison purposes a fresh cluster should be used to maintain accuracy.** + +### Remove All Sandbox-related Resources + +``` +make clean-e2e-resources +``` + +## Baseline Runs (Done by the Sandbox team) + +1. Install operators + + ``` + go run setup/main.go --users 1 --default 1 --custom 0 --username baseline --testname=baseline + ``` + +2. Run setup for 2000 users + + ``` + go run setup/main.go --users 2000 --default 2000 --custom 0 --username cupcake --testname=run1 + ``` diff --git a/setup/cmd/root.go b/setup/cmd/root.go index abd349150..2cc340d59 100644 --- a/setup/cmd/root.go +++ b/setup/cmd/root.go @@ -153,7 +153,7 @@ func setup(cmd *cobra.Command, _ []string) { // nolint:gocyclo token, err = auth.GetTokenFromOC() if err != nil { tokenRequestURI, err := auth.GetTokenRequestURI(cl) - errMsg := "a token is required to capture metrics, use oc login with token to log into the cluster. eg. `oc login --token= --server=`" + errMsg := "a token is required to capture metrics, use 'oc login' with token to log into the cluster. eg. `oc login --token= --server=`" if err != nil { term.Fatal(err, errMsg) } diff --git a/setup/terminal/terminal.go b/setup/terminal/terminal.go index 623dce303..7dc115d5b 100644 --- a/setup/terminal/terminal.go +++ b/setup/terminal/terminal.go @@ -16,14 +16,14 @@ import ( type Terminal interface { InOrStdin() io.Reader OutOrStdout() io.Writer - Debugf(msg string, args ...interface{}) + Debugf(msg string, args ...any) Info(msg string) - Infof(msg string, args ...interface{}) + Infof(msg string, args ...any) Error(err error, msg string) - Errorf(err error, msg string, args ...interface{}) + Errorf(err error, msg string, args ...any) Fatal(err error, msg string) - Fatalf(err error, msg string, args ...interface{}) - PromptBoolf(msg string, args ...interface{}) bool + Fatalf(err error, msg string, args ...any) + PromptBoolf(msg string, args ...any) bool AddPreFatalExitHook(func()) } @@ -57,7 +57,7 @@ type DefaultTerminal struct { } // Debugf prints a message (if verbose was enabled) -func (t *DefaultTerminal) Debugf(msg string, args ...interface{}) { +func (t *DefaultTerminal) Debugf(msg string, args ...any) { if !t.verbose { return } @@ -78,7 +78,7 @@ func (t *DefaultTerminal) Info(msg string) { } // Infof displays a message with the default color -func (t *DefaultTerminal) Infof(msg string, args ...interface{}) { +func (t *DefaultTerminal) Infof(msg string, args ...any) { if msg == "" { fmt.Fprintln(t.OutOrStdout(), "") return @@ -92,7 +92,7 @@ func (t *DefaultTerminal) Error(err error, msg string) { } // Errorf prints a message with the red color -func (t *DefaultTerminal) Errorf(err error, msg string, args ...interface{}) { +func (t *DefaultTerminal) Errorf(err error, msg string, args ...any) { color.New(color.FgRed).Fprintln(t.OutOrStdout(), fmt.Sprintf("%s: %s", fmt.Sprintf(msg, args...), err.Error())) // nolint:errcheck } @@ -106,7 +106,7 @@ func (t *DefaultTerminal) Fatal(err error, msg string) { } // Fatalf prints a message with the red color and exits the program with a `1` return code -func (t *DefaultTerminal) Fatalf(err error, msg string, args ...interface{}) { +func (t *DefaultTerminal) Fatalf(err error, msg string, args ...any) { defer os.Exit(1) for _, hook := range t.fatalExitHooks { hook() @@ -115,7 +115,7 @@ func (t *DefaultTerminal) Fatalf(err error, msg string, args ...interface{}) { } // PromptBoolf prints a message and waits for the user's boolean response -func (t *DefaultTerminal) PromptBoolf(msg string, args ...interface{}) bool { +func (t *DefaultTerminal) PromptBoolf(msg string, args ...any) bool { fmt.Fprintln(t.OutOrStdout(), fmt.Sprintf(msg, args...)) t.InOrStdin()