From 05a1a0ea5c82294b501b6e3c9ae7dbe84933e65d Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Mon, 7 Sep 2026 09:57:22 +0300 Subject: [PATCH 1/2] recovery: add the kubectl-gpurecovery plugin Kubectl plugin that helps with gpurecovery plan handling. Supports auto-complete so it's easier to list, approve, observe what's happening with the execution. Signed-off-by: Tuomas Katila --- Makefile | 23 +- RECOVERY.md | 7 +- cmd/kubectl-gpurecovery/apitypes.go | 73 ++++++ cmd/kubectl-gpurecovery/approvals.go | 132 ++++++++++ cmd/kubectl-gpurecovery/approve.go | 102 ++++++++ cmd/kubectl-gpurecovery/client.go | 118 +++++++++ cmd/kubectl-gpurecovery/confirm.go | 197 +++++++++++++++ cmd/kubectl-gpurecovery/events.go | 196 +++++++++++++++ cmd/kubectl-gpurecovery/format.go | 92 +++++++ cmd/kubectl-gpurecovery/main.go | 32 +++ cmd/kubectl-gpurecovery/messages.go | 121 ++++++++++ cmd/kubectl-gpurecovery/output.go | 75 ++++++ cmd/kubectl-gpurecovery/plans.go | 100 ++++++++ cmd/kubectl-gpurecovery/remove.go | 51 ++++ cmd/kubectl-gpurecovery/render_test.go | 317 +++++++++++++++++++++++++ cmd/kubectl-gpurecovery/root.go | 228 ++++++++++++++++++ go.mod | 2 +- 17 files changed, 1863 insertions(+), 3 deletions(-) create mode 100644 cmd/kubectl-gpurecovery/apitypes.go create mode 100644 cmd/kubectl-gpurecovery/approvals.go create mode 100644 cmd/kubectl-gpurecovery/approve.go create mode 100644 cmd/kubectl-gpurecovery/client.go create mode 100644 cmd/kubectl-gpurecovery/confirm.go create mode 100644 cmd/kubectl-gpurecovery/events.go create mode 100644 cmd/kubectl-gpurecovery/format.go create mode 100644 cmd/kubectl-gpurecovery/main.go create mode 100644 cmd/kubectl-gpurecovery/messages.go create mode 100644 cmd/kubectl-gpurecovery/output.go create mode 100644 cmd/kubectl-gpurecovery/plans.go create mode 100644 cmd/kubectl-gpurecovery/remove.go create mode 100644 cmd/kubectl-gpurecovery/render_test.go create mode 100644 cmd/kubectl-gpurecovery/root.go diff --git a/Makefile b/Makefile index 32c46cc..51d5a4f 100644 --- a/Makefile +++ b/Makefile @@ -114,9 +114,12 @@ vet: ## Run go vet against code. go vet ./... .PHONY: test +## The exclusion list drops the e2e suite (needs a cluster and real GPUs), the manager package +## (cmd, no tests — but cmd/kubectl-gpurecovery has them, hence the anchored '/cmd$$'), and the +## test/sample helpers. test: manifests generate fmt vet setup-envtest ## Run tests. chmod -R u+w $(LOCALBIN)/k8s || true ## by default binaries are not writable and cannot be removed - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v -e /e2e -e /cmd -e /test -e /samples) -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v -e /e2e -e '/cmd$$' -e /test -e /samples) -coverprofile cover.out # TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. # The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. @@ -175,6 +178,24 @@ lint-config: golangci-lint ## Verify golangci-lint linter configuration build: manifests generate fmt vet ## Build manager binary. go build -o bin/manager cmd/main.go +.PHONY: build-kubectl-plugin +build-kubectl-plugin: fmt vet ## Build kubectl-gpurecovery plugin binary to bin/kubectl-gpurecovery. + go build -o bin/kubectl-gpurecovery ./cmd/kubectl-gpurecovery/... + +.PHONY: install-kubectl-plugin +install-kubectl-plugin: build-kubectl-plugin ## Install kubectl-gpurecovery plugin to ~/.local/bin (must be on PATH). + install -m 0755 bin/kubectl-gpurecovery ~/.local/bin/kubectl-gpurecovery + @echo "Installed kubectl-gpurecovery to ~/.local/bin/kubectl-gpurecovery" + @echo "" + @echo "To enable tab completion for both 'kubectl-gpurecovery ' and 'kubectl gpurecovery '," + @echo "add the following to your ~/.bashrc (or ~/.zshrc for zsh):" + @echo "" + @echo " # kubectl plugin completion" + @echo " source <(kubectl completion bash)" + @echo " source <(kubectl-gpurecovery completion bash)" + @echo "" + @echo "Then reload your shell: source ~/.bashrc" + .PHONY: run run: manifests generate fmt vet ## Run a controller from your host. go run ./cmd/main.go diff --git a/RECOVERY.md b/RECOVERY.md index 48ec54c..85617e8 100644 --- a/RECOVERY.md +++ b/RECOVERY.md @@ -183,7 +183,7 @@ make install-kubectl-plugin # builds to bin/ and installs to ~/.local/bin make setup-completion # optional: shell completion kubectl gpurecovery plans -kubectl gpurecovery events +kubectl gpurecovery events [-o wide|yaml] kubectl gpurecovery messages kubectl gpurecovery approvals kubectl gpurecovery approve @@ -191,6 +191,11 @@ kubectl gpurecovery confirm [--persistent] kubectl gpurecovery remove ``` +`events` takes kubectl's `-o`: `-o wide` adds the approving approval, the recovery Job and +whatever is blocking the recovery (pods still on the node, ResourceClaims still reserving the +GPU) and stops truncating MESSAGE; `-o yaml` prints `status.events` verbatim, for piping into +`yq`/`jq`. + Plain `kubectl patch` works too, e.g.: ```sh diff --git a/cmd/kubectl-gpurecovery/apitypes.go b/cmd/kubectl-gpurecovery/apitypes.go new file mode 100644 index 0000000..f67a160 --- /dev/null +++ b/cmd/kubectl-gpurecovery/apitypes.go @@ -0,0 +1,73 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" +) + +// This file mirrors the vocabulary of the GPURecoveryPlan CRD +// (api/v1alpha1/gpurecoveryplan_types.go). The plugin talks to the API through the dynamic +// client, so nothing here is checked by the compiler — keep it in step with the CRD enums. + +// recoveryTypes are the values of the RecoveryType enum, i.e. what a +// status.events[].recoveryType.type or a selector's recoveryType can be. +var recoveryTypes = []string{"sbr", "slot", "amc", "reflash"} + +// overridableRecoveryTypes are the values accepted in spec.approvals[].override.recoveryType. +var overridableRecoveryTypes = []string{"sbr", "slot", "amc"} + +// approvableEventStates are the RecoveryEventState values in which an approval has an effect. +var approvableEventStates = map[string]bool{ + "waiting-approval": true, + "missing-firmware": true, + "failed": true, +} + +// validateOverride rejects an --override value the operator would silently ignore. +func validateOverride(override string) error { + if override == "" || slices.Contains(overridableRecoveryTypes, override) { + return nil + } + + if override == "reflash" { + return fmt.Errorf("--override reflash is not supported: the operator decides which events " + + "need a firmware reflash and ignores an override to it") + } + + return fmt.Errorf("invalid --override %q: must be one of %s", + override, strings.Join(overridableRecoveryTypes, ", ")) +} + +// completeStaticValues builds a completion function over a fixed value list. +func completeStaticValues(values []string) func(*cobra.Command, []string, string) ([]string, cobra.ShellCompDirective) { + return func(_ *cobra.Command, _ []string, toComplete string) ([]string, cobra.ShellCompDirective) { + var matches []string + + for _, v := range values { + if strings.HasPrefix(v, toComplete) { + matches = append(matches, v) + } + } + + return matches, cobra.ShellCompDirectiveNoFileComp + } +} diff --git a/cmd/kubectl-gpurecovery/approvals.go b/cmd/kubectl-gpurecovery/approvals.go new file mode 100644 index 0000000..687e3a9 --- /dev/null +++ b/cmd/kubectl-gpurecovery/approvals.go @@ -0,0 +1,132 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "sort" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +var approvalsCmd = &cobra.Command{ + Use: "approvals ", + Short: "List approvals configured on a plan", + Long: `List the approval entries in a plan's spec. + +OVERRIDE is the recovery type the approval substitutes for the operator's suggestion; +CONSUMED marks a non-persistent approval the operator has already acted upon, kept in the +list as an audit trail.`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: completePlanNames, + RunE: func(_ *cobra.Command, args []string) error { + cl, err := newDynamicClient() + if err != nil { + return err + } + + plan, err := getPlan(cl, args[0]) + if err != nil { + return err + } + + approvals := nestedSlice(plan.Object, "spec", "approvals") + if len(approvals) == 0 { + fmt.Println("No approvals.") + return nil + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "ID\tKIND\tTARGET\tOVERRIDE\tPERSISTENT\tCONSUMED\tCOMMENT") //nolint:errcheck + fmt.Fprintln(w, "──\t────\t──────\t────────\t──────────\t────────\t───────") //nolint:errcheck + + for _, raw := range approvals { + ap, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + id := nestedString(ap, "id") + kind, target, persistent := approvalSummary(ap) + consumed, _ := ap["consumed"].(bool) + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%v\t%v\t%s\n", //nolint:errcheck + id, kind, target, + orDash(nestedString(ap, "override", "recoveryType")), + persistent, consumed, + orDash(nestedString(ap, "comment"))) + } + + return w.Flush() + }, +} + +func approvalSummary(ap map[string]interface{}) (kind, target string, persistent bool) { + persistent, _ = ap["persistent"].(bool) + + if eventID, _ := ap["eventId"].(string); eventID != "" { + return "singular", eventID, persistent + } + + sel, ok := ap["selector"].(map[string]interface{}) + if !ok { + return "unknown", dash, persistent + } + + var parts []string + + if rt, _ := sel["recoveryType"].(string); rt != "" { + parts = append(parts, "type="+rt) + } + + if node, _ := sel["nodeName"].(string); node != "" { + parts = append(parts, "node="+node) + } + + if labels := selectorLabels(sel); labels != "" { + parts = append(parts, "labels="+labels) + } + + if len(parts) == 0 { + return "selector", "(any)", persistent + } + + return "selector", strings.Join(parts, " "), persistent +} + +// selectorLabels renders selector.nodeSelector as "k=v,k=v", sorted so the output is stable +// across invocations (map iteration order is not). +func selectorLabels(sel map[string]interface{}) string { + raw, ok := sel["nodeSelector"].(map[string]interface{}) + if !ok || len(raw) == 0 { + return "" + } + + pairs := make([]string, 0, len(raw)) + + for k, v := range raw { + s, _ := v.(string) + pairs = append(pairs, k+"="+s) + } + + sort.Strings(pairs) + + return strings.Join(pairs, ",") +} diff --git a/cmd/kubectl-gpurecovery/approve.go b/cmd/kubectl-gpurecovery/approve.go new file mode 100644 index 0000000..1871a85 --- /dev/null +++ b/cmd/kubectl-gpurecovery/approve.go @@ -0,0 +1,102 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" +) + +var ( + approveOverride string + approveComment string +) + +var approveCmd = &cobra.Command{ + Use: "approve ", + Short: "Approve a single recovery event by its ID", + Long: `Approve a specific recovery event. + +The event ID is copied from 'kubectl gpurecovery events '. +The operator generates an approval ID automatically. + +A failed event is terminal — nothing retries it on its own. Approving it again this way is what +starts another attempt; only an approval naming the event can do that, never a group approval. + +Use --override to run a different reset than the one the plan's defaultResetType picked, e.g. +an SBR on a card the platform's normal reset does not revive. The operator records the +original suggestion in status.events[].recoveryType.suggestedType.`, + Example: ` kubectl gpurecovery approve b580-recovery-plan evt-node03-slot-02-00-0 + + # Run an SBR on this one event instead of the plan's default reset + kubectl gpurecovery approve b580-recovery-plan evt-node03-slot-02-00-0 --override sbr`, + Args: cobra.ExactArgs(2), + ValidArgsFunction: completeEventIDs, + RunE: func(_ *cobra.Command, args []string) error { + planName := args[0] + eventID := args[1] + + if err := validateOverride(approveOverride); err != nil { + return err + } + + cl, err := newDynamicClient() + if err != nil { + return err + } + + approval := map[string]interface{}{ + "eventId": eventID, + } + + if approveOverride != "" { + approval["override"] = map[string]interface{}{ + "recoveryType": approveOverride, + } + } + + if approveComment != "" { + approval["comment"] = approveComment + } + + if err := addApproval(cl, planName, approval); err != nil { + return fmt.Errorf("approve event: %w", err) + } + + if approveOverride != "" { + fmt.Printf("Approved event %s on plan %s, overriding the recovery type to %s.\n", + eventID, planName, approveOverride) + } else { + fmt.Printf("Approved event %s on plan %s.\n", eventID, planName) + } + + return nil + }, +} + +func init() { + approveCmd.Flags().StringVar(&approveOverride, "override", "", + fmt.Sprintf("Run this reset instead of the plan's suggestion (one of %s)", + strings.Join(overridableRecoveryTypes, ", "))) + approveCmd.Flags().StringVar(&approveComment, "comment", "", + "Note recorded on the approval explaining why it was granted") + + // nolint:errcheck + approveCmd.RegisterFlagCompletionFunc("override", completeStaticValues(overridableRecoveryTypes)) +} diff --git a/cmd/kubectl-gpurecovery/client.go b/cmd/kubectl-gpurecovery/client.go new file mode 100644 index 0000000..7d271fe --- /dev/null +++ b/cmd/kubectl-gpurecovery/client.go @@ -0,0 +1,118 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/util/retry" +) + +func listOpts() metav1.ListOptions { return metav1.ListOptions{} } + +func getPlan(cl dynamic.Interface, name string) (*unstructured.Unstructured, error) { + plan, err := cl.Resource(gpuRecoveryPlanGVR).Get(context.Background(), name, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get plan %q: %w", name, err) + } + + return plan, nil +} + +// addApproval appends an approval entry to spec.approvals and updates the plan. +// approval must be a JSON-serialisable map matching RecoveryApproval fields. +func addApproval(cl dynamic.Interface, planName string, approval map[string]interface{}) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + plan, err := getPlan(cl, planName) + if err != nil { + return err + } + + approvals := nestedSlice(plan.Object, "spec", "approvals") + approvals = append(approvals, approval) + + if err := unstructured.SetNestedSlice(plan.Object, approvals, "spec", "approvals"); err != nil { + return fmt.Errorf("setting approvals: %w", err) + } + + _, err = cl.Resource(gpuRecoveryPlanGVR).Update( + context.Background(), plan, metav1.UpdateOptions{}, + ) + + return err + }) +} + +// removeApprovalByID removes the approval with the given ID from spec.approvals. +func removeApprovalByID(cl dynamic.Interface, planName, approvalID string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + plan, err := getPlan(cl, planName) + if err != nil { + return err + } + + approvals := nestedSlice(plan.Object, "spec", "approvals") + + kept := make([]interface{}, 0, len(approvals)) + found := false + + for _, raw := range approvals { + ap, ok := raw.(map[string]interface{}) + if !ok { + kept = append(kept, raw) + continue + } + + if id, _ := ap["id"].(string); id == approvalID { + found = true + continue + } + + kept = append(kept, raw) + } + + if !found { + return fmt.Errorf("approval %q not found in plan %q", approvalID, planName) + } + + if err := unstructured.SetNestedSlice(plan.Object, kept, "spec", "approvals"); err != nil { + return fmt.Errorf("setting approvals: %w", err) + } + + _, err = cl.Resource(gpuRecoveryPlanGVR).Update( + context.Background(), plan, metav1.UpdateOptions{}, + ) + return err + }) +} + +// nestedSlice safely extracts a []interface{} from a nested map path. +// Returns nil (empty slice) if the path doesn't exist. +func nestedSlice(obj map[string]interface{}, fields ...string) []interface{} { + s, _, _ := unstructured.NestedSlice(obj, fields...) + return s +} + +// nestedString safely extracts a string from a nested map path. +func nestedString(obj map[string]interface{}, fields ...string) string { + s, _, _ := unstructured.NestedString(obj, fields...) + return s +} diff --git a/cmd/kubectl-gpurecovery/confirm.go b/cmd/kubectl-gpurecovery/confirm.go new file mode 100644 index 0000000..83d2cc4 --- /dev/null +++ b/cmd/kubectl-gpurecovery/confirm.go @@ -0,0 +1,197 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "slices" + "strings" + + "github.com/spf13/cobra" +) + +var ( + confirmPersistent bool + confirmNodeName string + confirmNodeSelector map[string]string + confirmOverride string + confirmComment string +) + +var confirmCmd = &cobra.Command{ + Use: "confirm ", + Short: "Approve all current events of a given recovery type (group approval)", + Long: `Add a selector-based group approval for the given recovery type. + +This approves all currently waiting events that match. The approval is +marked as consumed (consumed=true) once the operator processes it, unless +--persistent is given. + +The recovery type is the type the events carry, which for resets is the plan's +spec.defaultResetType unless an event was overridden. It is not what the approval runs — use +--override for that. + +A group approval cannot restart a failed event; that needs +'kubectl gpurecovery approve '.`, + Example: ` # One-shot: approve all current slot-reset events + kubectl gpurecovery confirm b580-recovery-plan slot + + # Node-scoped one-shot + kubectl gpurecovery confirm b580-recovery-plan slot --node node07 + + # Label-scoped, and run an SBR instead of the suggested slot reset + kubectl gpurecovery confirm b580-recovery-plan slot --node-selector rack=a7 --override sbr + + # Persistent: auto-approve all future reflash events + kubectl gpurecovery confirm b580-recovery-plan reflash --persistent`, + Args: cobra.ExactArgs(2), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 { + return completePlanNames(cmd, args, toComplete) + } + + if len(args) == 1 { + return completeStaticValues(recoveryTypes)(cmd, args, toComplete) + } + + return nil, cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(_ *cobra.Command, args []string) error { + planName := args[0] + recoveryType := args[1] + + if !slices.Contains(recoveryTypes, recoveryType) { + return fmt.Errorf("invalid recovery type %q: must be one of %s", + recoveryType, strings.Join(recoveryTypes, ", ")) + } + + if err := validateOverride(confirmOverride); err != nil { + return err + } + + // The operator's own webhook rejects a nodeName+nodeSelector combination only if the + // labels are malformed, so catch the redundant pairing here where the intent is clear. + if confirmNodeName != "" && len(confirmNodeSelector) > 0 { + return fmt.Errorf("--node and --node-selector are mutually exclusive") + } + + cl, err := newDynamicClient() + if err != nil { + return err + } + + selector := map[string]interface{}{ + "recoveryType": recoveryType, + } + + if confirmNodeName != "" { + selector["nodeName"] = confirmNodeName + } + + if len(confirmNodeSelector) > 0 { + labels := make(map[string]interface{}, len(confirmNodeSelector)) + for k, v := range confirmNodeSelector { + labels[k] = v + } + + selector["nodeSelector"] = labels + } + + approval := map[string]interface{}{ + "selector": selector, + } + + if confirmPersistent { + approval["persistent"] = true + } + + if confirmOverride != "" { + approval["override"] = map[string]interface{}{ + "recoveryType": confirmOverride, + } + } + + if confirmComment != "" { + approval["comment"] = confirmComment + } + + if err := addApproval(cl, planName, approval); err != nil { + return fmt.Errorf("confirm group: %w", err) + } + + fmt.Printf("Added %s %s approval%s to plan %s%s.\n", + approvalKind(confirmPersistent), recoveryType, scopeSuffix(), planName, + overrideSuffix(confirmOverride)) + + return nil + }, +} + +func approvalKind(persistent bool) string { + if persistent { + return "persistent" + } + + return "one-shot" +} + +func scopeSuffix() string { + switch { + case confirmNodeName != "": + return " on node " + confirmNodeName + case len(confirmNodeSelector) > 0: + return " on nodes matching " + selectorLabels(map[string]interface{}{ + "nodeSelector": toUnstructuredMap(confirmNodeSelector), + }) + default: + return "" + } +} + +func overrideSuffix(override string) string { + if override == "" { + return "" + } + + return ", overriding the recovery type to " + override +} + +func toUnstructuredMap(m map[string]string) map[string]interface{} { + out := make(map[string]interface{}, len(m)) + for k, v := range m { + out[k] = v + } + + return out +} + +func init() { + confirmCmd.Flags().BoolVar(&confirmPersistent, "persistent", false, + "Keep the approval alive to auto-approve future matching events") + confirmCmd.Flags().StringVar(&confirmNodeName, "node", "", + "Restrict approval to events on a specific node") + confirmCmd.Flags().StringToStringVar(&confirmNodeSelector, "node-selector", nil, + "Restrict approval to events on nodes carrying all these labels (key=value,key=value)") + confirmCmd.Flags().StringVar(&confirmOverride, "override", "", + fmt.Sprintf("Run this reset instead of the events' suggestion (one of %s)", + strings.Join(overridableRecoveryTypes, ", "))) + confirmCmd.Flags().StringVar(&confirmComment, "comment", "", + "Note recorded on the approval explaining why it was granted") + + // nolint:errcheck + confirmCmd.RegisterFlagCompletionFunc("override", completeStaticValues(overridableRecoveryTypes)) +} diff --git a/cmd/kubectl-gpurecovery/events.go b/cmd/kubectl-gpurecovery/events.go new file mode 100644 index 0000000..81b4d46 --- /dev/null +++ b/cmd/kubectl-gpurecovery/events.go @@ -0,0 +1,196 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/spf13/cobra" +) + +var eventsOutput string + +// stateMessageWidth caps the MESSAGE column in the default (non-wide) listing so one long +// registry or API error does not push the rest of the table off screen. +const stateMessageWidth = 90 + +// The listing's columns, and the rule drawn under them. Kept out of the printing code because the +// wide variant's header is too long to fit a line there. +const ( + eventsHeader = "ID\tSTATE\tTYPE\tNODE\tBDF\tATTEMPTS\tUPDATED\tMESSAGE" + eventsHeaderRule = "──\t─────\t────\t────\t───\t────────\t───────\t───────" + + eventsWideHeader = "ID\tSTATE\tTYPE\tNODE\tBDF\tATTEMPTS\tUPDATED\tAPPROVAL\tJOB\tBLOCKED-BY\tMESSAGE" + eventsWideHeaderRule = "──\t─────\t────\t────\t───\t────────\t───────\t────────\t───\t──────────\t───────" +) + +var eventsCmd = &cobra.Command{ + Use: "events ", + Short: "List recovery events for a plan", + Long: `List the recovery events recorded in a plan's status. + +MESSAGE is status.events[].stateMessage: why the event is in the state it is in, where the +state alone does not say. It is empty whenever the state speaks for itself. + +ATTEMPTS is how many recovery Jobs the event has run, counted from status.events[].pastJobs +plus the Job in flight. Nothing retries a failed event on its own, so anything above one is an +event an admin approved again. + +-o wide additionally shows the approval that authorised the event, its recovery Job, and +anything currently holding the recovery back (pods blocking the drain, ResourceClaims still +reserving the GPU), and does not truncate MESSAGE. + +-o yaml prints status.events as it comes from the API, with the fields no column shows.`, + Example: ` kubectl gpurecovery events b580-recovery-plan + kubectl gpurecovery events b580-recovery-plan -o wide + kubectl gpurecovery events b580-recovery-plan -o yaml`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: completePlanNames, + RunE: func(_ *cobra.Command, args []string) error { + format, err := parseOutputFormat(eventsOutput) + if err != nil { + return err + } + + cl, err := newDynamicClient() + if err != nil { + return err + } + + plan, err := getPlan(cl, args[0]) + if err != nil { + return err + } + + events := nestedSlice(plan.Object, "status", "events") + + // A machine-readable format reports "no events" as an empty list, not as prose on + // stdout, so that piping it into a parser works on an idle plan too. + if format == outputYAML { + if events == nil { + events = []interface{}{} + } + + return printYAML(os.Stdout, events) + } + + if len(events) == 0 { + fmt.Println("No events.") + return nil + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + + if format == outputWide { + fmt.Fprintln(w, eventsWideHeader) //nolint:errcheck + fmt.Fprintln(w, eventsWideHeaderRule) //nolint:errcheck + } else { + fmt.Fprintln(w, eventsHeader) //nolint:errcheck + fmt.Fprintln(w, eventsHeaderRule) //nolint:errcheck + } + + for _, raw := range events { + ev, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + id := nestedString(ev, "id") + state := nestedString(ev, "state") + node := nestedString(ev, "nodeName") + bdf := nestedString(ev, "gpuBDF") + rt := recoveryTypeSummary(ev) + attempts := attemptCount(ev) + updated := formatTime(nestedString(ev, "lastUpdated")) + message := nestedString(ev, "stateMessage") + + if format != outputWide { + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%d\t%s\t%s\n", //nolint:errcheck + id, state, rt, node, bdf, attempts, updated, + orDash(truncate(message, stateMessageWidth))) + + continue + } + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\t%s\n", //nolint:errcheck + id, state, rt, node, bdf, attempts, updated, + orDash(nestedString(ev, "approvalId")), + orDash(nestedString(ev, "jobName")), + joinOrDash(blockers(ev)), + orDash(message)) + } + + return w.Flush() + }, +} + +// recoveryTypeSummary renders status.events[].recoveryType: the type being executed, plus the +// operator's original suggestion in parentheses when an approval override replaced it. +func recoveryTypeSummary(ev map[string]interface{}) string { + rt, ok := ev["recoveryType"].(map[string]interface{}) + if !ok { + return dash + } + + t, _ := rt["type"].(string) + if t == "" { + return dash + } + + if suggested, _ := rt["suggestedType"].(string); suggested != "" && suggested != t { + return fmt.Sprintf("%s (was %s)", t, suggested) + } + + return t +} + +// attemptCount reports how many recovery Jobs the event has run. The event carries no counter: +// pastJobs holds one entry per attempt that has reached a verdict, and jobName the attempt still +// in flight, which the operator only moves into pastJobs once its Job concludes. +func attemptCount(ev map[string]interface{}) int { + n := len(nestedSlice(ev, "pastJobs")) + + if nestedString(ev, "jobName") != "" { + n++ + } + + return n +} + +// blockers collects what is currently holding the recovery back: pods the drain is still +// waiting on and ResourceClaims that still reserve the GPU. +func blockers(ev map[string]interface{}) []string { + var out []string + + for _, field := range []string{"podsBlockingDrain", "claimsBlockingReset"} { + for _, raw := range nestedSlice(ev, field) { + if s, ok := raw.(string); ok && s != "" { + out = append(out, s) + } + } + } + + return out +} + +func init() { + addOutputFlag(eventsCmd, &eventsOutput, + "Output format: 'wide' adds the approval, Job and blocking pods/claims columns "+ + "and does not truncate MESSAGE, 'yaml' prints the raw status.events entries") +} diff --git a/cmd/kubectl-gpurecovery/format.go b/cmd/kubectl-gpurecovery/format.go new file mode 100644 index 0000000..67053c3 --- /dev/null +++ b/cmd/kubectl-gpurecovery/format.go @@ -0,0 +1,92 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + "time" +) + +// dash is printed for any field the CRD marks optional and the operator has not set, +// so that an empty column reads as "nothing recorded" rather than as a formatting bug. +const dash = "-" + +// compactDuration renders d in the abbreviated style kubectl uses for AGE columns. +func compactDuration(d time.Duration) string { + if d < 0 { + d = 0 + } + + switch { + case d < time.Minute: + return fmt.Sprintf("%ds", int(d.Seconds())) + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh%dm", int(d.Hours()), int(d.Minutes())%60) + default: + return fmt.Sprintf("%dd%dh", int(d.Hours())/24, int(d.Hours())%24) + } +} + +// formatTime renders an RFC3339 timestamp from a status field as an age ("5m ago"). +func formatTime(ts string) string { + if ts == "" { + return dash + } + + t, err := time.Parse(time.RFC3339, ts) + if err != nil { + return ts + } + + return compactDuration(time.Since(t)) + " ago" +} + +// truncate shortens s to max runes, ending in an ellipsis. Used for status.events[].stateMessage, +// which the CRD documents as truncated but still long enough to wrap a terminal. +func truncate(s string, maxLen int) string { + if maxLen <= 0 { + return s + } + + r := []rune(s) + if len(r) <= maxLen { + return s + } + + return string(r[:maxLen-1]) + "…" +} + +// orDash returns s, or the placeholder when s is empty. +func orDash(s string) string { + if s == "" { + return dash + } + + return s +} + +// joinOrDash renders a status list ("namespace/name" entries) as one column value. +func joinOrDash(items []string) string { + if len(items) == 0 { + return dash + } + + return strings.Join(items, ",") +} diff --git a/cmd/kubectl-gpurecovery/main.go b/cmd/kubectl-gpurecovery/main.go new file mode 100644 index 0000000..4ea78d5 --- /dev/null +++ b/cmd/kubectl-gpurecovery/main.go @@ -0,0 +1,32 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// kubectl-gpurecovery is a kubectl plugin for managing GPURecoveryPlan resources. +// Install it by placing the binary somewhere on PATH as "kubectl-gpurecovery". +// Usage: kubectl gpurecovery [flags] +package main + +import ( + "fmt" + "os" +) + +func main() { + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/kubectl-gpurecovery/messages.go b/cmd/kubectl-gpurecovery/messages.go new file mode 100644 index 0000000..3ff7c25 --- /dev/null +++ b/cmd/kubectl-gpurecovery/messages.go @@ -0,0 +1,121 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" +) + +var ( + messagesTail int + messagesEvent string +) + +var messagesCmd = &cobra.Command{ + Use: "messages ", + Short: "Show a plan's recent status messages", + Long: `Print status.messages: the plan's audit trail of what the operator did and why — +approvals matching, recovery types overridden, re-approvals, failures. + +It is a 50-entry ring shared by every event in the plan, printed oldest first, so ordinary +progress rotates older lines out. To find out why one event is stuck, read its MESSAGE in +'kubectl gpurecovery events ' instead; this is the history around it. + +Lines are free-form text, not structured records, so --event filters by matching the event ID +as a substring.`, + Example: ` kubectl gpurecovery messages b580-recovery-plan + + # Just the last ten lines + kubectl gpurecovery messages b580-recovery-plan --tail 10 + + # Only the lines naming one event + kubectl gpurecovery messages b580-recovery-plan --event evt-node03-slot-02-00-0`, + Args: cobra.ExactArgs(1), + ValidArgsFunction: completePlanNames, + RunE: func(_ *cobra.Command, args []string) error { + if messagesTail < 0 { + return fmt.Errorf("--tail must not be negative") + } + + cl, err := newDynamicClient() + if err != nil { + return err + } + + plan, err := getPlan(cl, args[0]) + if err != nil { + return err + } + + messages := planMessages(plan.Object, messagesEvent, messagesTail) + if len(messages) == 0 { + if messagesEvent != "" { + fmt.Printf("No messages mentioning %s.\n", messagesEvent) + } else { + fmt.Println("No messages.") + } + + return nil + } + + for _, msg := range messages { + fmt.Println(msg) + } + + return nil + }, +} + +// planMessages extracts status.messages, optionally keeping only the lines mentioning eventID +// and only the last tail entries. A tail of 0 means all of them. Filtering happens before the +// tail, so --tail counts the lines actually printed. +func planMessages(plan map[string]interface{}, eventID string, tail int) []string { + raw := nestedSlice(plan, "status", "messages") + out := make([]string, 0, len(raw)) + + for _, item := range raw { + msg, ok := item.(string) + if !ok { + continue + } + + if eventID != "" && !strings.Contains(msg, eventID) { + continue + } + + out = append(out, msg) + } + + if tail > 0 && len(out) > tail { + out = out[len(out)-tail:] + } + + return out +} + +func init() { + messagesCmd.Flags().IntVar(&messagesTail, "tail", 0, + "Show only the last N messages (0 shows all)") + messagesCmd.Flags().StringVar(&messagesEvent, "event", "", + "Show only messages mentioning this event ID") + + // nolint:errcheck + messagesCmd.RegisterFlagCompletionFunc("event", completeEventIDsForFlag) +} diff --git a/cmd/kubectl-gpurecovery/output.go b/cmd/kubectl-gpurecovery/output.go new file mode 100644 index 0000000..d567614 --- /dev/null +++ b/cmd/kubectl-gpurecovery/output.go @@ -0,0 +1,75 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "strings" + + "github.com/spf13/cobra" + "sigs.k8s.io/yaml" +) + +// outputFormat is a -o/--output value. The plugin implements the subset of kubectl's formats its +// own output has a meaning for: the default table, the same table with the columns a narrow +// terminal cannot fit, and the raw API data behind the table. +type outputFormat string + +const ( + outputTable outputFormat = "" + outputWide outputFormat = "wide" + outputYAML outputFormat = "yaml" +) + +// outputFormats are the accepted -o values, in the order the flag's error message and its shell +// completion list them. +var outputFormats = []string{"wide", "yaml"} + +// parseOutputFormat validates a raw -o value. An empty value is the default table, so that an +// empty -o argument behaves like omitting the flag rather than failing. +func parseOutputFormat(s string) (outputFormat, error) { + switch f := outputFormat(s); f { + case outputTable, outputWide, outputYAML: + return f, nil + default: + return outputTable, fmt.Errorf("unsupported output format %q: must be one of %s", + s, strings.Join(outputFormats, ", ")) + } +} + +// addOutputFlag registers -o/--output on cmd together with the completion for its values. +func addOutputFlag(cmd *cobra.Command, target *string, usage string) { + cmd.Flags().StringVarP(target, "output", "o", "", usage) + + // nolint:errcheck + cmd.RegisterFlagCompletionFunc("output", completeStaticValues(outputFormats)) +} + +// printYAML writes v as YAML to out. v must hold only the JSON-compatible types the dynamic +// client produces (map[string]interface{}, []interface{}, string, bool, int64, float64), which +// is what sigs.k8s.io/yaml can round-trip through JSON. +func printYAML(out io.Writer, v interface{}) error { + buf, err := yaml.Marshal(v) + if err != nil { + return fmt.Errorf("marshalling YAML: %w", err) + } + + _, err = out.Write(buf) + + return err +} diff --git a/cmd/kubectl-gpurecovery/plans.go b/cmd/kubectl-gpurecovery/plans.go new file mode 100644 index 0000000..f13c99a --- /dev/null +++ b/cmd/kubectl-gpurecovery/plans.go @@ -0,0 +1,100 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "os" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" +) + +var plansCmd = &cobra.Command{ + Use: "plans", + Short: "List all GPURecoveryPlan resources in the cluster", + Long: `List the GPURecoveryPlans in the cluster. + +STATE is status.state: "error" means at least one event needs human intervention, +"active" that a recovery is in flight or waiting for an approval, "idle" that there is +nothing to do. WAITING counts the events that an approval would start.`, + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, _ []string) error { + cl, err := newDynamicClient() + if err != nil { + return err + } + + list, err := cl.Resource(gpuRecoveryPlanGVR).List(context.Background(), listOpts()) + if err != nil { + return fmt.Errorf("listing GPURecoveryPlans: %w", err) + } + + if len(list.Items) == 0 { + fmt.Println("No GPURecoveryPlans found.") + return nil + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "NAME\tDEVICE-ID\tRESET\tSTATE\tEVENTS\tWAITING\tAPPROVALS\tAGE") //nolint:errcheck + fmt.Fprintln(w, "────\t─────────\t─────\t─────\t──────\t───────\t─────────\t───") //nolint:errcheck + + for i := range list.Items { + item := &list.Items[i] + + events := nestedSlice(item.Object, "status", "events") + approvals := nestedSlice(item.Object, "spec", "approvals") + + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d\t%d\t%d\t%s\n", //nolint:errcheck + item.GetName(), + orDash(nestedString(item.Object, "spec", "deviceId")), + orDash(nestedString(item.Object, "spec", "defaultResetType")), + orDash(nestedString(item.Object, "status", "state")), + len(events), + countWaitingEvents(events), + len(approvals), + compactDuration(time.Since(item.GetCreationTimestamp().Time))) + } + + return w.Flush() + }, +} + +// countWaitingEvents counts the events an approval would act on, i.e. the ones the operator +// considers approvable (see approvableEventStates). +func countWaitingEvents(events []interface{}) int { + n := 0 + + for _, raw := range events { + ev, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + if state, _ := ev["state"].(string); approvableEventStates[state] { + n++ + } + } + + return n +} + +func init() { + rootCmd.AddCommand(plansCmd) +} diff --git a/cmd/kubectl-gpurecovery/remove.go b/cmd/kubectl-gpurecovery/remove.go new file mode 100644 index 0000000..5d201e5 --- /dev/null +++ b/cmd/kubectl-gpurecovery/remove.go @@ -0,0 +1,51 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var removeCmd = &cobra.Command{ + Use: "remove ", + Short: "Remove an approval from a plan", + Long: `Remove a specific approval entry by its ID. + +Use 'kubectl gpurecovery approvals ' to list approval IDs.`, + Example: ` kubectl gpurecovery remove b580-recovery-plan app-a4af`, + Args: cobra.ExactArgs(2), + ValidArgsFunction: completeApprovalIDs, + RunE: func(_ *cobra.Command, args []string) error { + planName := args[0] + approvalID := args[1] + + cl, err := newDynamicClient() + if err != nil { + return err + } + + if err := removeApprovalByID(cl, planName, approvalID); err != nil { + return fmt.Errorf("remove approval: %w", err) + } + + fmt.Printf("Removed approval %s from plan %s.\n", approvalID, planName) + + return nil + }, +} diff --git a/cmd/kubectl-gpurecovery/render_test.go b/cmd/kubectl-gpurecovery/render_test.go new file mode 100644 index 0000000..f885ca9 --- /dev/null +++ b/cmd/kubectl-gpurecovery/render_test.go @@ -0,0 +1,317 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "testing" +) + +// The fixtures below are the JSON shape of GPURecoveryPlan as the API server serves it. They +// exist because the plugin reads the CRD through the dynamic client: a renamed or moved status +// field is not a compile error here, it silently prints "-". + +func TestRecoveryTypeSummary(t *testing.T) { + for _, tc := range []struct { + name string + ev map[string]interface{} + want string + }{ + { + name: "reset", + ev: map[string]interface{}{"recoveryType": map[string]interface{}{"type": "slot"}}, + want: "slot", + }, + { + name: "reflash", + ev: map[string]interface{}{"recoveryType": map[string]interface{}{"type": "reflash"}}, + want: "reflash", + }, + { + name: "overridden reports the operator's original suggestion", + ev: map[string]interface{}{"recoveryType": map[string]interface{}{ + "type": "sbr", "suggestedType": "slot", + }}, + want: "sbr (was slot)", + }, + { + name: "suggestedType equal to type is not an override", + ev: map[string]interface{}{"recoveryType": map[string]interface{}{ + "type": "amc", "suggestedType": "amc", + }}, + want: "amc", + }, + { + name: "missing recoveryType", + ev: map[string]interface{}{}, + want: dash, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := recoveryTypeSummary(tc.ev); got != tc.want { + t.Errorf("recoveryTypeSummary() = %q, want %q", got, tc.want) + } + }) + } +} + +func TestAttemptCount(t *testing.T) { + for _, tc := range []struct { + name string + ev map[string]interface{} + want int + }{ + { + name: "no Job has run yet", + ev: map[string]interface{}{}, + want: 0, + }, + { + name: "first attempt in flight is counted before it concludes", + ev: map[string]interface{}{"jobName": "recovery-evt-a-0"}, + want: 1, + }, + { + // The operator clears jobName as it appends to pastJobs, so a concluded attempt is + // counted once, not twice. + name: "concluded attempt", + ev: map[string]interface{}{"pastJobs": []interface{}{"recovery-evt-a-0"}}, + want: 1, + }, + { + name: "re-approved event running its second attempt", + ev: map[string]interface{}{ + "pastJobs": []interface{}{"recovery-evt-a-0"}, + "jobName": "recovery-evt-a-1", + }, + want: 2, + }, + } { + t.Run(tc.name, func(t *testing.T) { + if got := attemptCount(tc.ev); got != tc.want { + t.Errorf("attemptCount() = %d, want %d", got, tc.want) + } + }) + } +} + +func TestBlockers(t *testing.T) { + ev := map[string]interface{}{ + "podsBlockingDrain": []interface{}{"default/trainer-0"}, + "claimsBlockingReset": []interface{}{"default/claim-a", "default/claim-b"}, + } + + want := "default/trainer-0,default/claim-a,default/claim-b" + if got := joinOrDash(blockers(ev)); got != want { + t.Errorf("blockers() = %q, want %q", got, want) + } + + if got := joinOrDash(blockers(map[string]interface{}{})); got != dash { + t.Errorf("blockers() on an unblocked event = %q, want %q", got, dash) + } +} + +func TestApprovalSummary(t *testing.T) { + for _, tc := range []struct { + name string + ap map[string]interface{} + wantKind string + wantTarget string + wantPersistent bool + }{ + { + name: "event approval", + ap: map[string]interface{}{"eventId": "evt-node03-slot-02-00-0"}, + wantKind: "singular", + wantTarget: "evt-node03-slot-02-00-0", + }, + { + name: "selector with every field", + ap: map[string]interface{}{ + "persistent": true, + "selector": map[string]interface{}{ + "recoveryType": "slot", + "nodeName": "node07", + "nodeSelector": map[string]interface{}{"rack": "a7", "zone": "b"}, + }, + }, + wantKind: "selector", + wantTarget: "type=slot node=node07 labels=rack=a7,zone=b", + wantPersistent: true, + }, + { + name: "empty selector matches anything", + ap: map[string]interface{}{"selector": map[string]interface{}{}}, + wantKind: "selector", + wantTarget: "(any)", + }, + } { + t.Run(tc.name, func(t *testing.T) { + kind, target, persistent := approvalSummary(tc.ap) + if kind != tc.wantKind || target != tc.wantTarget || persistent != tc.wantPersistent { + t.Errorf("approvalSummary() = (%q, %q, %v), want (%q, %q, %v)", + kind, target, persistent, tc.wantKind, tc.wantTarget, tc.wantPersistent) + } + }) + } +} + +func TestCountWaitingEvents(t *testing.T) { + events := []interface{}{ + map[string]interface{}{"state": "waiting-approval"}, + map[string]interface{}{"state": "missing-firmware"}, + map[string]interface{}{"state": "failed"}, + map[string]interface{}{"state": "blocked"}, + map[string]interface{}{"state": "draining"}, + map[string]interface{}{"state": "in-progress"}, + map[string]interface{}{"state": "succeeded"}, + } + + if got := countWaitingEvents(events); got != 3 { + t.Errorf("countWaitingEvents() = %d, want 3", got) + } +} + +func TestPlanMessages(t *testing.T) { + plan := map[string]interface{}{ + "status": map[string]interface{}{ + "messages": []interface{}{ + "Event evt-node03-slot-02-00-0 detected", + "Event evt-node07-amc-04-00-0 detected", + "Event evt-node03-slot-02-00-0: reset type overridden from slot to sbr via approval app-a4af", + "Event evt-node03-slot-02-00-0 succeeded", + }, + }, + } + + all := planMessages(plan, "", 0) + if len(all) != 4 { + t.Fatalf("planMessages() returned %d messages, want 4", len(all)) + } + + if all[0] != "Event evt-node03-slot-02-00-0 detected" { + t.Errorf("planMessages() is not oldest-first: first line is %q", all[0]) + } + + if got := planMessages(plan, "", 2); len(got) != 2 || got[1] != all[3] { + t.Errorf("planMessages(tail=2) = %v, want the last two lines", got) + } + + if got := planMessages(plan, "evt-node07-amc-04-00-0", 0); len(got) != 1 { + t.Errorf("planMessages(event=...) = %v, want the one matching line", got) + } + + // The filter runs before the tail, so --tail counts printed lines rather than scanned ones. + if got := planMessages(plan, "evt-node03-slot-02-00-0", 2); len(got) != 2 || got[1] != all[3] { + t.Errorf("planMessages(event=..., tail=2) = %v, want the last two matching lines", got) + } + + if got := planMessages(map[string]interface{}{}, "", 0); len(got) != 0 { + t.Errorf("planMessages() on a plan with no status = %v, want empty", got) + } +} + +func TestParseOutputFormat(t *testing.T) { + for _, tc := range []struct { + in string + want outputFormat + wantErr bool + }{ + {in: "", want: outputTable}, + {in: "wide", want: outputWide}, + {in: "yaml", want: outputYAML}, + // kubectl accepts these; this plugin does not, and says so rather than falling back + // to the table and printing something the caller cannot parse. + {in: "json", wantErr: true}, + {in: "jsonpath={.id}", wantErr: true}, + {in: "YAML", wantErr: true}, + {in: "w", wantErr: true}, + } { + got, err := parseOutputFormat(tc.in) + + if tc.wantErr { + if err == nil { + t.Errorf("parseOutputFormat(%q) = %q, want an error", tc.in, got) + } + + continue + } + + if err != nil { + t.Errorf("parseOutputFormat(%q) = %v, want %q", tc.in, err, tc.want) + continue + } + + if got != tc.want { + t.Errorf("parseOutputFormat(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestPrintYAML(t *testing.T) { + events := []interface{}{ + map[string]interface{}{ + "id": "evt-node03-slot-02-00-0", + "state": "waiting-approval", + "pastJobs": []interface{}{"recovery-evt-node03-slot-02-00-0-0"}, + "recoveryType": map[string]interface{}{"type": "slot"}, + }, + } + + var buf bytes.Buffer + if err := printYAML(&buf, events); err != nil { + t.Fatalf("printYAML() = %v", err) + } + + want := `- id: evt-node03-slot-02-00-0 + pastJobs: + - recovery-evt-node03-slot-02-00-0-0 + recoveryType: + type: slot + state: waiting-approval +` + + if buf.String() != want { + t.Errorf("printYAML() =\n%s\nwant\n%s", buf.String(), want) + } + + // An idle plan has no events; a parser reading the output must see an empty list rather + // than the "null" a nil slice would marshal to. + buf.Reset() + + if err := printYAML(&buf, []interface{}{}); err != nil { + t.Fatalf("printYAML() on no events = %v", err) + } + + if buf.String() != "[]\n" { + t.Errorf("printYAML() on no events = %q, want %q", buf.String(), "[]\n") + } +} + +func TestValidateOverride(t *testing.T) { + for _, override := range []string{"", "sbr", "slot", "amc"} { + if err := validateOverride(override); err != nil { + t.Errorf("validateOverride(%q) = %v, want nil", override, err) + } + } + + for _, override := range []string{"reflash", "flr", "SBR"} { + if err := validateOverride(override); err == nil { + t.Errorf("validateOverride(%q) = nil, want an error", override) + } + } +} diff --git a/cmd/kubectl-gpurecovery/root.go b/cmd/kubectl-gpurecovery/root.go new file mode 100644 index 0000000..3d33ea3 --- /dev/null +++ b/cmd/kubectl-gpurecovery/root.go @@ -0,0 +1,228 @@ +/* +Copyright 2026 Intel Corporation. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + kubeconfig string + kubeContext string +) + +var gpuRecoveryPlanGVR = schema.GroupVersionResource{ + Group: "intel.com", + Version: "v1alpha1", + Resource: "gpurecoveryplans", +} + +var rootCmd = &cobra.Command{ + Use: "kubectl-gpurecovery", + Short: "Manage GPU recovery plans and events", + Long: "A kubectl plugin for inspecting and approving GPURecoveryPlan events.", +} + +func init() { + rootCmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", "", + "Path to kubeconfig (defaults to KUBECONFIG env then ~/.kube/config)") + rootCmd.PersistentFlags().StringVar(&kubeContext, "context", "", + "Kubernetes context to use") + + rootCmd.AddCommand(eventsCmd) + rootCmd.AddCommand(messagesCmd) + rootCmd.AddCommand(approvalsCmd) + rootCmd.AddCommand(approveCmd) + rootCmd.AddCommand(confirmCmd) + rootCmd.AddCommand(removeCmd) + rootCmd.AddCommand(completionCmd) +} + +func newDynamicClient() (dynamic.Interface, error) { + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + if kubeconfig != "" { + loadingRules.ExplicitPath = kubeconfig + } + + overrides := &clientcmd.ConfigOverrides{} + if kubeContext != "" { + overrides.CurrentContext = kubeContext + } + + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + loadingRules, overrides, + ).ClientConfig() + if err != nil { + return nil, fmt.Errorf("building kubeconfig: %w", err) + } + + return dynamic.NewForConfig(cfg) +} + +// completePlanNames returns a ValidArgsFunction that completes GPURecoveryPlan names. +func completePlanNames(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) > 0 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + cl, err := newDynamicClient() + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + list, err := cl.Resource(gpuRecoveryPlanGVR).List(context.Background(), listOpts()) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + var names []string + + for _, item := range list.Items { + if name := item.GetName(); strings.HasPrefix(name, toComplete) { + names = append(names, name) + } + } + + return names, cobra.ShellCompDirectiveNoFileComp +} + +// completeEventIDs is a ValidArgsFunction that completes the IDs of events an approval would +// act on (see approvableEventStates) from the plan given as args[0]. +func completeEventIDs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 { + return completePlanNames(cmd, args, toComplete) + } + + if len(args) > 1 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return completeEventIDsInPlan(args[0], toComplete, approvableEventStates) +} + +// completeEventIDsForFlag completes an --event flag value with every event ID in the plan named +// by args[0], regardless of state: a flag that filters history is not restricted to the events +// something can still be done about. +func completeEventIDsForFlag(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return completeEventIDsInPlan(args[0], toComplete, nil) +} + +// completeEventIDsInPlan lists the status.events[].id values in planName that start with +// toComplete. A nil states map accepts every state. +func completeEventIDsInPlan(planName, toComplete string, states map[string]bool) ([]string, cobra.ShellCompDirective) { + cl, err := newDynamicClient() + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + plan, err := getPlan(cl, planName) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + var ids []string + + for _, raw := range nestedSlice(plan.Object, "status", "events") { + ev, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + if states != nil { + if state, _ := ev["state"].(string); !states[state] { + continue + } + } + + if id, _ := ev["id"].(string); strings.HasPrefix(id, toComplete) { + ids = append(ids, id) + } + } + + return ids, cobra.ShellCompDirectiveNoFileComp +} + +// completeApprovalIDs returns a ValidArgsFunction that completes approval IDs +// from the plan given as args[0]. +func completeApprovalIDs(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if len(args) == 0 { + return completePlanNames(cmd, args, toComplete) + } + + if len(args) > 1 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + cl, err := newDynamicClient() + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + plan, err := getPlan(cl, args[0]) + if err != nil { + return nil, cobra.ShellCompDirectiveError + } + + approvals := nestedSlice(plan.Object, "spec", "approvals") + var ids []string + + for _, raw := range approvals { + ap, ok := raw.(map[string]interface{}) + if !ok { + continue + } + + if id, _ := ap["id"].(string); strings.HasPrefix(id, toComplete) { + ids = append(ids, id) + } + } + + return ids, cobra.ShellCompDirectiveNoFileComp +} + +var completionCmd = &cobra.Command{ + Use: "completion [bash|zsh|fish|powershell]", + Short: "Generate shell completion script", + ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + switch args[0] { + case "bash": + return rootCmd.GenBashCompletion(os.Stdout) + case "zsh": + return rootCmd.GenZshCompletion(os.Stdout) + case "fish": + return rootCmd.GenFishCompletion(os.Stdout, true) + case "powershell": + return rootCmd.GenPowerShellCompletionWithDesc(os.Stdout) + default: + return fmt.Errorf("unsupported shell: %s", args[0]) + } + }, +} diff --git a/go.mod b/go.mod index d308118..5caee82 100644 --- a/go.mod +++ b/go.mod @@ -76,7 +76,7 @@ require ( github.com/prometheus/common v0.70.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/sirupsen/logrus v1.9.4 // indirect - github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect From 853a6685d52b561b5ce2e5e90429e139fd9e0638 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Wed, 9 Sep 2026 19:29:37 +0300 Subject: [PATCH 2/2] workflow: build kubectl-plugin Signed-off-by: Tuomas Katila --- .github/workflows/validate-common.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validate-common.yaml b/.github/workflows/validate-common.yaml index f5fabff..937ef11 100644 --- a/.github/workflows/validate-common.yaml +++ b/.github/workflows/validate-common.yaml @@ -59,6 +59,7 @@ jobs: go-version-file: go.mod check-latest: true cache: false + - run: make build-kubectl-plugin - run: make build - run: IMG=ghcr.io/intel/intel-gpu-base-operator:trivy make operator-build - name: Run Trivy for operator image (json)