recovery: add the kubectl-gpurecovery plugin

.github/workflows/validate-common.yaml

@@ -59,6 +59,7 @@ jobs:
           go-version-file: go.mod
           check-latest: true
           cache: false
+      - run: make build-kubectl-plugin
       - run: make build
       - run: IMG=ghcr.io/intel/intel-gpu-base-operator:trivy make operator-build
       - name: Run Trivy for operator image (json)

Makefile

@@ -114,9 +114,12 @@ vet: ## Run go vet against code.
 	go vet ./...
 
 .PHONY: test
+## The exclusion list drops the e2e suite (needs a cluster and real GPUs), the manager package
+## (cmd, no tests — but cmd/kubectl-gpurecovery has them, hence the anchored '/cmd$$'), and the
+## test/sample helpers.
 test: manifests generate fmt vet setup-envtest ## Run tests.
 	chmod -R u+w $(LOCALBIN)/k8s || true ## by default binaries are not writable and cannot be removed
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v -e /e2e -e /cmd -e /test -e /samples) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v -e /e2e -e '/cmd$$' -e /test -e /samples) -coverprofile cover.out
 
 # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'.
 # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally.
@@ -175,6 +178,24 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration
 build: manifests generate fmt vet ## Build manager binary.
 	go build -o bin/manager cmd/main.go
 
+.PHONY: build-kubectl-plugin
+build-kubectl-plugin: fmt vet ## Build kubectl-gpurecovery plugin binary to bin/kubectl-gpurecovery.
+	go build -o bin/kubectl-gpurecovery ./cmd/kubectl-gpurecovery/...
+
+.PHONY: install-kubectl-plugin
+install-kubectl-plugin: build-kubectl-plugin ## Install kubectl-gpurecovery plugin to ~/.local/bin (must be on PATH).
+	install -m 0755 bin/kubectl-gpurecovery ~/.local/bin/kubectl-gpurecovery
+	@echo "Installed kubectl-gpurecovery to ~/.local/bin/kubectl-gpurecovery"
+	@echo ""
+	@echo "To enable tab completion for both 'kubectl-gpurecovery <tab>' and 'kubectl gpurecovery <tab>',"
+	@echo "add the following to your ~/.bashrc (or ~/.zshrc for zsh):"
+	@echo ""
+	@echo "  # kubectl plugin completion"
+	@echo "  source <(kubectl completion bash)"
+	@echo "  source <(kubectl-gpurecovery completion bash)"
+	@echo ""
+	@echo "Then reload your shell: source ~/.bashrc"
+
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./cmd/main.go

RECOVERY.md

@@ -183,14 +183,19 @@ make install-kubectl-plugin   # builds to bin/ and installs to ~/.local/bin
 make setup-completion         # optional: shell completion
 
 kubectl gpurecovery plans
-kubectl gpurecovery events <plan>
+kubectl gpurecovery events <plan> [-o wide|yaml]
 kubectl gpurecovery messages <plan>
 kubectl gpurecovery approvals <plan>
 kubectl gpurecovery approve <plan> <event-id>
 kubectl gpurecovery confirm <plan> <recovery-type> [--persistent]
 kubectl gpurecovery remove <plan> <approval-id>
 ```
 
+`events` takes kubectl's `-o`: `-o wide` adds the approving approval, the recovery Job and
+whatever is blocking the recovery (pods still on the node, ResourceClaims still reserving the
+GPU) and stops truncating MESSAGE; `-o yaml` prints `status.events` verbatim, for piping into
+`yq`/`jq`.
+
 Plain `kubectl patch` works too, e.g.:
 
 ```sh

cmd/kubectl-gpurecovery/apitypes.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2026 Intel Corporation. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+// This file mirrors the vocabulary of the GPURecoveryPlan CRD
+// (api/v1alpha1/gpurecoveryplan_types.go). The plugin talks to the API through the dynamic
+// client, so nothing here is checked by the compiler — keep it in step with the CRD enums.
+
+// recoveryTypes are the values of the RecoveryType enum, i.e. what a
+// status.events[].recoveryType.type or a selector's recoveryType can be.
+var recoveryTypes = []string{"sbr", "slot", "amc", "reflash"}
+
+// overridableRecoveryTypes are the values accepted in spec.approvals[].override.recoveryType.
+var overridableRecoveryTypes = []string{"sbr", "slot", "amc"}
+
+// approvableEventStates are the RecoveryEventState values in which an approval has an effect.
+var approvableEventStates = map[string]bool{
+	"waiting-approval": true,
+	"missing-firmware": true,
+	"failed":           true,
+}
+
+// validateOverride rejects an --override value the operator would silently ignore.
+func validateOverride(override string) error {
+	if override == "" || slices.Contains(overridableRecoveryTypes, override) {
+		return nil
+	}
+
+	if override == "reflash" {
+		return fmt.Errorf("--override reflash is not supported: the operator decides which events " +
+			"need a firmware reflash and ignores an override to it")
+	}
+
+	return fmt.Errorf("invalid --override %q: must be one of %s",
+		override, strings.Join(overridableRecoveryTypes, ", "))
+}
+
+// completeStaticValues builds a completion function over a fixed value list.
+func completeStaticValues(values []string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) {
+	return func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		var matches []string
+
+		for _, v := range values {
+			if strings.HasPrefix(v, toComplete) {
+				matches = append(matches, v)
+			}
+		}
+
+		return matches, cobra.ShellCompDirectiveNoFileComp
+	}
+}

cmd/kubectl-gpurecovery/approvals.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2026 Intel Corporation. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"github.com/spf13/cobra"
+)
+
+var approvalsCmd = &cobra.Command{
+	Use:   "approvals <plan>",
+	Short: "List approvals configured on a plan",
+	Long: `List the approval entries in a plan's spec.
+
+OVERRIDE is the recovery type the approval substitutes for the operator's suggestion;
+CONSUMED marks a non-persistent approval the operator has already acted upon, kept in the
+list as an audit trail.`,
+	Args:              cobra.ExactArgs(1),
+	ValidArgsFunction: completePlanNames,
+	RunE: func(_ *cobra.Command, args []string) error {
+		cl, err := newDynamicClient()
+		if err != nil {
+			return err
+		}
+
+		plan, err := getPlan(cl, args[0])
+		if err != nil {
+			return err
+		}
+
+		approvals := nestedSlice(plan.Object, "spec", "approvals")
+		if len(approvals) == 0 {
+			fmt.Println("No approvals.")
+			return nil
+		}
+
+		w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0)
+		fmt.Fprintln(w, "ID\tKIND\tTARGET\tOVERRIDE\tPERSISTENT\tCONSUMED\tCOMMENT") //nolint:errcheck
+		fmt.Fprintln(w, "──\t────\t──────\t────────\t──────────\t────────\t───────") //nolint:errcheck
+
+		for _, raw := range approvals {
+			ap, ok := raw.(map[string]interface{})
+			if !ok {
+				continue
+			}
+
+			id := nestedString(ap, "id")
+			kind, target, persistent := approvalSummary(ap)
+			consumed, _ := ap["consumed"].(bool)
+
+			fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%v\t%v\t%s\n", //nolint:errcheck
+				id, kind, target,
+				orDash(nestedString(ap, "override", "recoveryType")),
+				persistent, consumed,
+				orDash(nestedString(ap, "comment")))
+		}
+
+		return w.Flush()
+	},
+}
+
+func approvalSummary(ap map[string]interface{}) (kind, target string, persistent bool) {
+	persistent, _ = ap["persistent"].(bool)
+
+	if eventID, _ := ap["eventId"].(string); eventID != "" {
+		return "singular", eventID, persistent
+	}
+
+	sel, ok := ap["selector"].(map[string]interface{})
+	if !ok {
+		return "unknown", dash, persistent
+	}
+
+	var parts []string
+
+	if rt, _ := sel["recoveryType"].(string); rt != "" {
+		parts = append(parts, "type="+rt)
+	}
+
+	if node, _ := sel["nodeName"].(string); node != "" {
+		parts = append(parts, "node="+node)
+	}
+
+	if labels := selectorLabels(sel); labels != "" {
+		parts = append(parts, "labels="+labels)
+	}
+
+	if len(parts) == 0 {
+		return "selector", "(any)", persistent
+	}
+
+	return "selector", strings.Join(parts, " "), persistent
+}
+
+// selectorLabels renders selector.nodeSelector as "k=v,k=v", sorted so the output is stable
+// across invocations (map iteration order is not).
+func selectorLabels(sel map[string]interface{}) string {
+	raw, ok := sel["nodeSelector"].(map[string]interface{})
+	if !ok || len(raw) == 0 {
+		return ""
+	}
+
+	pairs := make([]string, 0, len(raw))
+
+	for k, v := range raw {
+		s, _ := v.(string)
+		pairs = append(pairs, k+"="+s)
+	}
+
+	sort.Strings(pairs)
+
+	return strings.Join(pairs, ",")
+}

cmd/kubectl-gpurecovery/approve.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2026 Intel Corporation. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	approveOverride string
+	approveComment  string
+)
+
+var approveCmd = &cobra.Command{
+	Use:   "approve <plan> <event-id>",
+	Short: "Approve a single recovery event by its ID",
+	Long: `Approve a specific recovery event.
+
+The event ID is copied from 'kubectl gpurecovery events <plan>'.
+The operator generates an approval ID automatically.
+
+A failed event is terminal — nothing retries it on its own. Approving it again this way is what
+starts another attempt; only an approval naming the event can do that, never a group approval.
+
+Use --override to run a different reset than the one the plan's defaultResetType picked, e.g.
+an SBR on a card the platform's normal reset does not revive. The operator records the
+original suggestion in status.events[].recoveryType.suggestedType.`,
+	Example: `  kubectl gpurecovery approve b580-recovery-plan evt-node03-slot-02-00-0
+
+  # Run an SBR on this one event instead of the plan's default reset
+  kubectl gpurecovery approve b580-recovery-plan evt-node03-slot-02-00-0 --override sbr`,
+	Args:              cobra.ExactArgs(2),
+	ValidArgsFunction: completeEventIDs,
+	RunE: func(_ *cobra.Command, args []string) error {
+		planName := args[0]
+		eventID := args[1]
+
+		if err := validateOverride(approveOverride); err != nil {
+			return err
+		}
+
+		cl, err := newDynamicClient()
+		if err != nil {
+			return err
+		}
+
+		approval := map[string]interface{}{
+			"eventId": eventID,
+		}
+
+		if approveOverride != "" {
+			approval["override"] = map[string]interface{}{
+				"recoveryType": approveOverride,
+			}
+		}
+
+		if approveComment != "" {
+			approval["comment"] = approveComment
+		}
+
+		if err := addApproval(cl, planName, approval); err != nil {
+			return fmt.Errorf("approve event: %w", err)
+		}
+
+		if approveOverride != "" {
+			fmt.Printf("Approved event %s on plan %s, overriding the recovery type to %s.\n",
+				eventID, planName, approveOverride)
+		} else {
+			fmt.Printf("Approved event %s on plan %s.\n", eventID, planName)
+		}
+
+		return nil
+	},
+}
+
+func init() {
+	approveCmd.Flags().StringVar(&approveOverride, "override", "",
+		fmt.Sprintf("Run this reset instead of the plan's suggestion (one of %s)",
+			strings.Join(overridableRecoveryTypes, ", ")))
+	approveCmd.Flags().StringVar(&approveComment, "comment", "",
+		"Note recorded on the approval explaining why it was granted")
+
+	// nolint:errcheck
+	approveCmd.RegisterFlagCompletionFunc("override", completeStaticValues(overridableRecoveryTypes))
+}