Xpumd restart functionality

README.md

@@ -206,6 +206,7 @@ Applies to both DP and DRA unless noted. Thresholds that are exceeded mark the G
 |`spec.xpu.monitoringResource`|Set XPUMD resource for Device Plugin use.|`monitoring`|
 |`spec.xpu.configMapOverride`|Name of a ConfigMap in the operator namespace containing a custom OpenTelemetry Collector `config.yaml`|—|
 |`spec.xpu.affinity`|Optional `k8s.io/api/core/v1` `Affinity` object applied to the XPU manager daemon set|—|
+|`spec.xpu.restartOnDeviceRecovery`|When to restart a node's XPU Manager pod so it can monitor a GPU its container cannot reach. `OnRecoveredDevice`, `Always` or `Disabled`.|`OnRecoveredDevice`|
 
 #### Kueue (`spec.kueue`)
 

api/v1alpha1/clusterpolicy_types.go

@@ -163,6 +163,13 @@ type XpuManagerSpec struct {
 	// Set optional affinities for XPU pods
 	// +optional
 	Affinity *v1.Affinity `json:"affinity,omitempty"`
+
+	// RestartOnDeviceRecovery controls whether the operator restarts a node's XPU Manager pod so
+	// that it can monitor a GPU its container cannot currently reach.
+	// +kubebuilder:validation:Enum=OnRecoveredDevice;Always;Disabled
+	// +kubebuilder:default:=OnRecoveredDevice
+	// +optional
+	RestartOnDeviceRecovery XpumRestartMode `json:"restartOnDeviceRecovery,omitempty"`
 }
 
 // RegistryTLSSpec configures TLS behavior for accessing container image registries.
@@ -267,6 +274,22 @@ type BuildArg struct {
 	Value string `json:"value"`
 }
 
+// XpumRestartMode selects when the operator replaces a node's XPU Manager pod to give it a GPU its
+// container cannot reach.
+type XpumRestartMode string
+
+const (
+	// XpumRestartOnRecoveredDevice restarts the pod for a usable GPU its container was never
+	// given a device node for, and leaves a re-enumerated one to XPU Manager's own rescan.
+	XpumRestartOnRecoveredDevice XpumRestartMode = "OnRecoveredDevice"
+
+	// XpumRestartAlways also restarts when a device the container does hold re-enumerates.
+	XpumRestartAlways XpumRestartMode = "Always"
+
+	// XpumRestartDisabled disables both the record and the restart.
+	XpumRestartDisabled XpumRestartMode = "Disabled"
+)
+
 // ClusterPolicyStatus defines the observed state of ClusterPolicy.
 type ClusterPolicyStatus struct {
 	DevicePluginStatus string   `json:"devicePluginStatus,omitempty"`

charts/gpu-base-operator-policy/README.md

@@ -47,6 +47,7 @@ See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_h
 | xpu.logLevel | 2 | XPU manager log level. |
 | xpu.monitoringResource | monitoring | Monitoring resource for XPUMD with device plugin. |
 | xpu.configMapOverride | "" | Override the default XPUM configuration ConfigMap name. |
+| xpu.restartOnDeviceRecovery | OnRecoveredDevice | When to restart a node's XPUMD pod so it can monitor a GPU its container cannot reach: `OnRecoveredDevice`, `Always` or `Disabled`. |
 | kueue.equalResources | [] | List of ClusterQueue configurations. |
 | pullSecret | null | Image pull secret. |
 | nodeSelector | {} | Node selector for scheduling pods. |

charts/gpu-base-operator-policy/templates/clusterpolicy.yaml

@@ -73,6 +73,9 @@ spec:
 {{- if .Values.xpu.configMapOverride }}
     configMapOverride: {{ .Values.xpu.configMapOverride | default "" }}
 {{- end }}
+{{- if .Values.xpu.restartOnDeviceRecovery }}
+    restartOnDeviceRecovery: {{ .Values.xpu.restartOnDeviceRecovery | quote }}
+{{- end }}
 
 {{- if .Values.kernelModule }}
   kernelModule:

charts/gpu-base-operator-policy/values.yaml

@@ -48,6 +48,8 @@ xpu:
 #       - matchExpressions:
 #         - key: gpu.intel.com/xpumd-deny-node
 #           operator: DoesNotExist
+  # Restart a node's XPU Manager pod so it can monitor a GPU its container cannot reach.
+  restartOnDeviceRecovery: OnRecoveredDevice
 
 kueue:
   equalResources:

charts/gpu-base-operator/crds/clusterpolicies.yaml

@@ -3191,6 +3191,16 @@ spec:
                     - xe_monitoring
                     - monitoring
                     type: string
+                  restartOnDeviceRecovery:
+                    default: OnRecoveredDevice
+                    description: |-
+                      RestartOnDeviceRecovery controls whether the operator restarts a node's XPU Manager pod so
+                      that it can monitor a GPU its container cannot currently reach.
+                    enum:
+                    - OnRecoveredDevice
+                    - Always
+                    - Disabled
+                    type: string
                 type: object
             required:
             - resourceRegistration

charts/gpu-base-operator/templates/namespaced_role.yaml

@@ -26,6 +26,14 @@ rules:
   - pods/log
   verbs:
   - get
+# Patching pods is how XpumDeviceRefreshReconciler records, on each XPU Manager pod, which of its
+# node's GPUs its container can actually use. Kept in step with config/rbac/namespaced_role.yaml.
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources:

cmd/main.go

@@ -375,6 +375,13 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "GPURecoveryPlan")
 		os.Exit(1)
 	}
+	if err := (&controller.XpumDeviceRefreshReconciler{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr, copts); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "XpumDeviceRefresh")
+		os.Exit(1)
+	}
 
 	// nolint:goconst
 	if os.Getenv("DISABLE_WEBHOOKS") != "true" {

config/crd/bases/intel.com_clusterpolicies.yaml

@@ -3191,6 +3191,16 @@ spec:
                     - xe_monitoring
                     - monitoring
                     type: string
+                  restartOnDeviceRecovery:
+                    default: OnRecoveredDevice
+                    description: |-
+                      RestartOnDeviceRecovery controls whether the operator restarts a node's XPU Manager pod so
+                      that it can monitor a GPU its container cannot currently reach.
+                    enum:
+                    - OnRecoveredDevice
+                    - Always
+                    - Disabled
+                    type: string
                 type: object
             required:
             - resourceRegistration

config/rbac/namespaced_role.yaml

@@ -25,6 +25,16 @@ rules:
   - pods/log
   verbs:
   - get
+# Patching pods is how XpumDeviceRefreshReconciler records, on each XPU Manager pod, which of its
+# node's GPUs its container can actually use. Granted here rather than through a kubebuilder marker
+# because those all merge into the cluster-wide role, and this write only ever targets the operator's
+# own namespace. Reading and deleting pods is cluster-wide already, for the firmware update jobs.
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - patch
 - apiGroups:
   - ""
   resources: