From d779ae65cd67e9cbdf8257e3f9032294d2a6f322 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Sun, 27 Sep 2026 12:38:44 -0500 Subject: [PATCH 1/3] fix(cdi): bound refresh restart attempts Signed-off-by: Kevin Rajan --- deployments/systemd/nvidia-cdi-refresh.service | 9 ++++++--- tests/e2e/nvidia-cdi-refresh_test.go | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/deployments/systemd/nvidia-cdi-refresh.service b/deployments/systemd/nvidia-cdi-refresh.service index 974c7b8f8..361c904b2 100644 --- a/deployments/systemd/nvidia-cdi-refresh.service +++ b/deployments/systemd/nvidia-cdi-refresh.service @@ -18,12 +18,15 @@ ConditionPathExists=|/usr/bin/nvidia-smi ConditionPathExists=|/usr/sbin/nvidia-smi ConditionPathExists=|/usr/lib/wsl/lib/nvidia-smi ConditionPathExists=/usr/bin/nvidia-ctk -# Limit the number of successive restarts to 5 in 10 seconds. -StartLimitBurst=5 -StartLimitIntervalSec=10s [Service] Type=oneshot +# The packaged 10-container-engines.conf drop-in bounds each activation at 90s. +# Keep the retry window longer than five bounded attempts plus RestartSec so the +# sixth start is rejected even when driver probing takes the full timeout. +# Use the legacy Service spelling for compatibility with older systemd releases. +StartLimitBurst=5 +StartLimitInterval=10min # Values from Environment will be replaced if defined in EnvironmentFile Environment=NVIDIA_CTK_CDI_OUTPUT_FILE_PATH=/var/run/cdi/nvidia.yaml EnvironmentFile=-/etc/nvidia-container-toolkit/nvidia-cdi-refresh.env diff --git a/tests/e2e/nvidia-cdi-refresh_test.go b/tests/e2e/nvidia-cdi-refresh_test.go index 0bee922ca..b2ab19070 100644 --- a/tests/e2e/nvidia-cdi-refresh_test.go +++ b/tests/e2e/nvidia-cdi-refresh_test.go @@ -109,6 +109,18 @@ EOF echo "nvidia-cdi-refresh.service is not ordered before docker.service" exit 1 fi + if ! systemctl show nvidia-cdi-refresh.service -p TimeoutStartUSec | grep -q "TimeoutStartUSec=1min 30s"; then + echo "nvidia-cdi-refresh.service does not have the expected 90s start timeout" + exit 1 + fi + if ! systemctl cat nvidia-cdi-refresh.service | grep -q "^StartLimitBurst=5$"; then + echo "nvidia-cdi-refresh.service does not have the expected restart burst" + exit 1 + fi + if ! systemctl cat nvidia-cdi-refresh.service | grep -q "^StartLimitInterval=10min$"; then + echo "nvidia-cdi-refresh.service does not have the expected restart interval" + exit 1 + fi ` nvidiaCdiRefreshFileExistsTemplate = ` From 88761720e1138cf6c16ab199e88405143b061abb Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:50:19 -0500 Subject: [PATCH 2/3] test(cdi): check effective refresh restart limits Signed-off-by: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> --- tests/e2e/nvidia-cdi-refresh-template_test.go | 149 ++++++++++++++++++ tests/e2e/nvidia-cdi-refresh_test.go | 11 +- 2 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 tests/e2e/nvidia-cdi-refresh-template_test.go diff --git a/tests/e2e/nvidia-cdi-refresh-template_test.go b/tests/e2e/nvidia-cdi-refresh-template_test.go new file mode 100644 index 000000000..0aae1b22d --- /dev/null +++ b/tests/e2e/nvidia-cdi-refresh-template_test.go @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package e2e + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestNvidiaCdiRefreshEffectiveRestartLimits(t *testing.T) { + const expectedProperties = "StartLimitBurst=5\nStartLimitIntervalUSec=10min\nTimeoutStartUSec=1min 30s\n" + const matchingUnitText = "[Service]\nStartLimitBurst=5\nStartLimitInterval=10min\n" + for _, tc := range []struct { + name string + properties string + unitText string + showExit string + wantError string + }{ + { + name: "effective limits match", + properties: expectedProperties, + unitText: matchingUnitText, + }, + { + name: "equivalent modern unit spelling", + properties: expectedProperties, + unitText: "[Unit]\nStartLimitBurst=5\nStartLimitIntervalSec=600s\n", + }, + { + name: "drop-in disables the burst limit", + properties: strings.ReplaceAll(expectedProperties, "StartLimitBurst=5", "StartLimitBurst=0"), + unitText: matchingUnitText, + wantError: "does not have the expected restart burst", + }, + { + name: "drop-in shortens the restart window", + properties: strings.ReplaceAll(expectedProperties, "StartLimitIntervalUSec=10min", "StartLimitIntervalUSec=10s"), + unitText: matchingUnitText, + wantError: "does not have the expected restart interval", + }, + { + name: "unsupported interval property", + properties: strings.ReplaceAll(expectedProperties, "StartLimitIntervalUSec=10min\n", ""), + unitText: matchingUnitText, + wantError: "does not have the expected restart interval", + }, + { + name: "timeout only has matching prefix", + properties: strings.ReplaceAll(expectedProperties, "TimeoutStartUSec=1min 30s", "TimeoutStartUSec=1min 30s 500ms"), + unitText: matchingUnitText, + wantError: "does not have the expected 90s start timeout", + }, + { + name: "systemctl fails despite matching output", + properties: expectedProperties, + unitText: matchingUnitText, + showExit: "1", + wantError: "Could not read nvidia-cdi-refresh.service restart settings", + }, + { + name: "missing burst property", + properties: strings.ReplaceAll(expectedProperties, "StartLimitBurst=5\n", ""), + unitText: matchingUnitText, + wantError: "does not have the expected restart burst", + }, + { + name: "missing timeout property", + properties: strings.ReplaceAll(expectedProperties, "TimeoutStartUSec=1min 30s\n", ""), + unitText: matchingUnitText, + wantError: "does not have the expected 90s start timeout", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + dropIn := filepath.Join(dir, "10-container-engines.conf") + if err := os.WriteFile(dropIn, nil, 0600); err != nil { + t.Fatal(err) + } + // Never call the host's systemctl. Model raw unit text independently + // from the effective properties, as with a later overriding drop-in. + stub := `#!/bin/sh +command=$1 +shift +[ "$1" = "nvidia-cdi-refresh.service" ] || exit 99 +shift +case "$command" in +cat) + [ "$#" -eq 0 ] || exit 99 + printf '%s' "$UNIT_TEXT" + ;; +show) + [ "$#" -gt 0 ] || exit 99 + check_settings=0 + while [ "$#" -gt 0 ]; do + [ "$1" = "-p" ] && [ "$#" -ge 2 ] || exit 99 + case "$2" in + Before) printf '%s\n' 'Before=docker.service containerd.service crio.service' ;; + TimeoutStartUSec|StartLimitBurst|StartLimitIntervalUSec) + printf '%s' "$EFFECTIVE_PROPERTIES" | grep "^$2=" || true + check_settings=1 + ;; + *) exit 99 ;; + esac + shift 2 + done + [ "$check_settings" -eq 0 ] || exit "${SHOW_EXIT:-0}" + ;; +*) exit 99 ;; +esac +` + if err := os.WriteFile(filepath.Join(dir, "systemctl"), []byte(stub), 0700); err != nil { + t.Fatal(err) + } + script := strings.ReplaceAll(nvidiaCdiRefreshOrderingDropInInstalledTemplate, + "/lib/systemd/system/nvidia-cdi-refresh.service.d/10-container-engines.conf", dropIn) + cmd := exec.Command("bash", "-c", script) + cmd.Env = append(os.Environ(), + "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"), + "UNIT_TEXT="+tc.unitText, + "EFFECTIVE_PROPERTIES="+tc.properties, + "SHOW_EXIT="+tc.showExit) + output, err := cmd.CombinedOutput() + if (err != nil) != (tc.wantError != "") { + t.Fatalf("template error = %v, want error = %q; output:\n%s", err, tc.wantError, output) + } + if tc.wantError != "" && !strings.Contains(string(output), tc.wantError) { + t.Fatalf("template output = %q, want diagnostic containing %q", output, tc.wantError) + } + }) + } +} diff --git a/tests/e2e/nvidia-cdi-refresh_test.go b/tests/e2e/nvidia-cdi-refresh_test.go index b2ab19070..8689b2c5d 100644 --- a/tests/e2e/nvidia-cdi-refresh_test.go +++ b/tests/e2e/nvidia-cdi-refresh_test.go @@ -109,15 +109,20 @@ EOF echo "nvidia-cdi-refresh.service is not ordered before docker.service" exit 1 fi - if ! systemctl show nvidia-cdi-refresh.service -p TimeoutStartUSec | grep -q "TimeoutStartUSec=1min 30s"; then + # Read the effective settings: unit text can be ignored or overridden by a drop-in. + if ! settings=$(systemctl show nvidia-cdi-refresh.service -p TimeoutStartUSec -p StartLimitBurst -p StartLimitIntervalUSec); then + echo "Could not read nvidia-cdi-refresh.service restart settings" + exit 1 + fi + if ! printf '%s\n' "$settings" | grep -qx "TimeoutStartUSec=1min 30s"; then echo "nvidia-cdi-refresh.service does not have the expected 90s start timeout" exit 1 fi - if ! systemctl cat nvidia-cdi-refresh.service | grep -q "^StartLimitBurst=5$"; then + if ! printf '%s\n' "$settings" | grep -qx "StartLimitBurst=5"; then echo "nvidia-cdi-refresh.service does not have the expected restart burst" exit 1 fi - if ! systemctl cat nvidia-cdi-refresh.service | grep -q "^StartLimitInterval=10min$"; then + if ! printf '%s\n' "$settings" | grep -qx "StartLimitIntervalUSec=10min"; then echo "nvidia-cdi-refresh.service does not have the expected restart interval" exit 1 fi From 9e6dce4e46009c8bd8ec0dbcb9422833a6593315 Mon Sep 17 00:00:00 2001 From: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> Date: Wed, 30 Sep 2026 23:56:08 -0500 Subject: [PATCH 3/3] ci: run hermetic CDI refresh assertion regressions Signed-off-by: Kevin Rajan <7121943+kvnloo@users.noreply.github.com> --- .github/workflows/golang.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index c504d23a8..73ecdf9fa 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -75,6 +75,9 @@ jobs: - name: Run unit tests and generate coverage report run: make coverage + - name: Check CDI refresh assertions without a systemd host + run: go -C tests test ./e2e -run '^TestNvidiaCdiRefreshEffectiveRestartLimits$' -count=1 + - name: Upload to Coveralls continue-on-error: true uses: coverallsapp/github-action@v2