Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions cmd/nerdctl/container/container_create.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,10 @@ func createOptions(cmd *cobra.Command) (types.ContainerCreateOptions, error) {
if err != nil {
return opt, err
}
opt.HealthStartInterval, err = cmd.Flags().GetDuration("health-start-interval")
if err != nil {
return opt, err
}
opt.NoHealthcheck, err = cmd.Flags().GetBool("no-healthcheck")
if err != nil {
return opt, err
Expand Down
116 changes: 116 additions & 0 deletions cmd/nerdctl/container/container_health_check_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ func TestContainerHealthCheckDefaults(t *testing.T) {
assert.Equal(t, hc.Timeout, 30*time.Second, "expected default timeout of 30s")
assert.Equal(t, hc.Retries, 3, "expected default retries of 3")
assert.Equal(t, hc.StartPeriod, 0*time.Second, "expected default start period of 0s")
assert.Equal(t, hc.StartInterval, 5*time.Second, "expected default start interval of 5s")

// Verify the command was set correctly
assert.DeepEqual(t, hc.Test, []string{"CMD-SHELL", "echo healthy"})
Expand All @@ -206,6 +207,7 @@ func TestContainerHealthCheckDefaults(t *testing.T) {
"--health-timeout", "15s",
"--health-retries", "5",
"--health-start-period", "10s",
"--health-start-interval", "3s",
testutil.CommonImage, "sleep", nerdtest.Infinity)
nerdtest.EnsureContainerStarted(helpers, data.Identifier())
},
Expand Down Expand Up @@ -234,6 +236,7 @@ func TestContainerHealthCheckDefaults(t *testing.T) {
assert.Equal(t, hc.Timeout, 15*time.Second, "expected custom timeout of 15s")
assert.Equal(t, hc.Retries, 5, "expected custom retries of 5")
assert.Equal(t, hc.StartPeriod, 10*time.Second, "expected custom start period of 10s")
assert.Equal(t, hc.StartInterval, 3*time.Second, "expected custom start interval of 3s")

// Verify the command was set correctly
assert.DeepEqual(t, hc.Test, []string{"CMD-SHELL", "echo custom"})
Expand Down Expand Up @@ -652,6 +655,11 @@ func TestContainerHealthCheckAdvance(t *testing.T) {
"--health-cmd", "exit 1",
"--health-timeout", "10s",
"--health-retries", "3",
// Probes are now throttled to run no more often than --health-interval
// (see shouldRunProbe), which otherwise defaults to 30s: shorter than
// this test's manual invocations are apart, so each of them would
// otherwise be skipped as "not due yet" instead of actually probing.
Comment on lines +658 to +661

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When the throttle skips a manual nerdctl container healthcheck, the command exits 0 and prints nothing, so it looks exactly like a successful probe, which is confusing. We should at least log the skip at Info or Warn level.

"--health-interval", "1s",
testutil.CommonImage, "sleep", nerdtest.Infinity)
nerdtest.EnsureContainerStarted(helpers, data.Identifier())
},
Expand Down Expand Up @@ -751,6 +759,114 @@ func TestContainerHealthCheckAdvance(t *testing.T) {
testCase.Run(t)
}

// TestContainerHealthCheckStartInterval covers --health-start-interval cadence behavior:
// probes ticking at the faster start-interval cadence while inside the start period, and
// falling back to the regular health-interval cadence once the start period ends. Kept
// separate from TestContainerHealthCheckAdvance to stay under the function-length lint limit.
func TestContainerHealthCheckStartInterval(t *testing.T) {
testCase := nerdtest.Setup()

// Docker CLI does not provide a standalone healthcheck command.
testCase.Require = require.Not(nerdtest.Docker)

// Skip systemd tests in rootless environment to bypass dbus permission issues
if rootlessutil.IsRootless() {
t.Skip("systemd healthcheck tests are skipped in rootless environment")
}

testCase.SubTests = []*test.Case{
{
Description: "Health check probes at start-interval cadence within the start period",
Setup: func(data test.Data, helpers test.Helpers) {
helpers.Ensure("run", "-d", "--name", data.Identifier(),
"--health-cmd", "exit 1",
"--health-interval", "60s",
"--health-start-period", "30s",
"--health-start-interval", "1s",
testutil.CommonImage, "sleep", nerdtest.Infinity)
nerdtest.EnsureContainerStarted(helpers, data.Identifier())
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
helpers.Ensure("container", "healthcheck", data.Identifier())
// Longer than --health-start-interval (1s) but much shorter than
// --health-interval (60s): this tick must still run because we are
// still within --health-start-period.
time.Sleep(2 * time.Second)
helpers.Ensure("container", "healthcheck", data.Identifier())
return helpers.Command("inspect", data.Identifier())
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 0,
Output: expect.All(func(stdout string, t tig.T) {
inspect := nerdtest.InspectContainer(helpers, data.Identifier())
h := inspect.State.Health
debug, _ := json.MarshalIndent(h, "", " ")
t.Log(string(debug))
assert.Assert(t, h != nil, "expected health state")
// health-cmd always fails, so unhealthy results are ignored and we
// remain in the start period workflow throughout.
assert.Equal(t, h.Status, healthcheck.Starting)
// At least our two manual ticks must have run, since each was
// spaced beyond --health-start-interval. We can only assert a
// lower bound: the container's own background timer also ticks
// at the --health-start-interval cadence during the start
// period, and may legitimately fire once more inside this same
// window, adding an extra probe beyond the two we triggered.
assert.Assert(t, len(h.Log) >= 2,
"expected both manual ticks to run: each was spaced beyond --health-start-interval")
}),
}
},
},
{
Description: "Health check falls back to health-interval cadence once the start period ends",
Setup: func(data test.Data, helpers test.Helpers) {
helpers.Ensure("run", "-d", "--name", data.Identifier(),
"--health-cmd", "exit 0",
"--health-interval", "60s",
"--health-start-period", "5s",
"--health-start-interval", "1s",
testutil.CommonImage, "sleep", nerdtest.Infinity)
nerdtest.EnsureContainerStarted(helpers, data.Identifier())
},
Cleanup: func(data test.Data, helpers test.Helpers) {
helpers.Anyhow("rm", "-f", data.Identifier())
},
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
// First tick always runs and, since health-cmd succeeds, immediately
// exits the start period (first healthy result).
helpers.Ensure("container", "healthcheck", data.Identifier())
// Second tick arrives well within --health-start-interval (1s), but the
// start period already ended, so --health-interval (60s) now applies and
// this tick must be skipped.
helpers.Ensure("container", "healthcheck", data.Identifier())
return helpers.Command("inspect", data.Identifier())
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: 0,
Output: expect.All(func(stdout string, t tig.T) {
inspect := nerdtest.InspectContainer(helpers, data.Identifier())
h := inspect.State.Health
debug, _ := json.MarshalIndent(h, "", " ")
t.Log(string(debug))
assert.Assert(t, h != nil, "expected health state")
assert.Equal(t, h.Status, healthcheck.Healthy)
assert.Equal(t, len(h.Log), 1,
"expected the second tick to be throttled by --health-interval after the start period ended")
}),
}
},
},
}

testCase.Run(t)
}

func TestHealthCheck_SystemdIntegration_Basic(t *testing.T) {
testCase := nerdtest.Setup()
testCase.Require = require.Not(nerdtest.Docker)
Expand Down
1 change: 1 addition & 0 deletions cmd/nerdctl/container/container_run.go
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ func setCreateFlags(cmd *cobra.Command) {
cmd.Flags().Duration("health-timeout", 0, "Maximum time to allow one check to run; 0 uses the image value or 30s when unset there too")
cmd.Flags().Int("health-retries", 0, "Consecutive failures needed to report unhealthy; 0 uses the image value or 3 when unset there too")
cmd.Flags().Duration("health-start-period", 0, "Start period for the container to initialize before starting health-retries countdown")
cmd.Flags().Duration("health-start-interval", 0, "Time between running the check during the start period; 0 uses the image value or 5s when unset there too")
cmd.Flags().Bool("no-healthcheck", false, "Disable any container-specified HEALTHCHECK")

// #region env flags
Expand Down
40 changes: 27 additions & 13 deletions cmd/nerdctl/container/container_run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -937,14 +937,15 @@ func TestRunHealthcheckFlags(t *testing.T) {
testCase.Require = require.Not(nerdtest.Rootless)

testCases := []struct {
name string
args []string
shouldFail bool
expectTest []string
expectRetries int
expectInterval time.Duration
expectTimeout time.Duration
expectStartPeriod time.Duration
name string
args []string
shouldFail bool
expectTest []string
expectRetries int
expectInterval time.Duration
expectTimeout time.Duration
expectStartPeriod time.Duration
expectStartInterval time.Duration
}{
{
name: "Valid_full_config",
Expand All @@ -954,12 +955,14 @@ func TestRunHealthcheckFlags(t *testing.T) {
"--health-timeout", "5s",
"--health-retries", "3",
"--health-start-period", "2s",
"--health-start-interval", "1s",
},
expectTest: []string{"CMD-SHELL", "curl -f http://localhost || exit 1"},
expectInterval: 30 * time.Second,
expectTimeout: 5 * time.Second,
expectRetries: 3,
expectStartPeriod: 2 * time.Second,
expectTest: []string{"CMD-SHELL", "curl -f http://localhost || exit 1"},
expectInterval: 30 * time.Second,
expectTimeout: 5 * time.Second,
expectRetries: 3,
expectStartPeriod: 2 * time.Second,
expectStartInterval: 1 * time.Second,
},
{
name: "No_healthcheck",
Expand Down Expand Up @@ -996,6 +999,14 @@ func TestRunHealthcheckFlags(t *testing.T) {
},
shouldFail: true,
},
{
name: "Negative_start_interval",
args: []string{
"--health-cmd", "true",
"--health-start-interval", "-1s",
},
shouldFail: true,
},
{
name: "Invalid_timeout_format",
args: []string{
Expand Down Expand Up @@ -1067,6 +1078,9 @@ func TestRunHealthcheckFlags(t *testing.T) {
if tc.expectStartPeriod > 0 {
assert.Equal(t, hc.StartPeriod, tc.expectStartPeriod)
}
if tc.expectStartInterval > 0 {
assert.Equal(t, hc.StartInterval, tc.expectStartInterval)
}
},
),
}
Expand Down
6 changes: 5 additions & 1 deletion cmd/nerdctl/helpers/flagutil.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ func ValidateHealthcheckFlags(options types.ContainerCreateOptions) error {
options.HealthInterval != 0 ||
options.HealthTimeout != 0 ||
options.HealthRetries != 0 ||
options.HealthStartPeriod != 0
options.HealthStartPeriod != 0 ||
options.HealthStartInterval != 0

if options.NoHealthcheck {
if options.HealthCmd != "" || healthFlagsSet {
Expand All @@ -73,6 +74,9 @@ func ValidateHealthcheckFlags(options types.ContainerCreateOptions) error {
if options.HealthStartPeriod < 0 {
return fmt.Errorf("--health-start-period cannot be negative")
}
if options.HealthStartInterval < 0 {
return fmt.Errorf("--health-start-interval cannot be negative")
}
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions docs/command-reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,7 @@ Health check flags:
- :whale: `--health-timeout`: Time to wait before considering the check failed (e.g., 5s)
- :whale: `--health-retries`: Number of failures before container is considered unhealthy
- :whale: `--health-start-period`: Start period for the container to initialize before starting health-retries countdown
- :whale: `--health-start-interval`: Time between running the check during the start period (e.g., 5s)
- :whale: `--no-healthcheck`: Disable any health checks defined by image or CLI

Logging flags:
Expand Down Expand Up @@ -475,8 +476,7 @@ IPFS flags:

Unimplemented `docker run` flags:
`--device-cgroup-rule`, `--disable-content-trust`,
`--health-start-interval`, `--link*`, `--storage-opt`,
`--volume-driver`
`--link*`, `--storage-opt`, `--volume-driver`

### :whale: nerdctl exec

Expand Down
1 change: 0 additions & 1 deletion docs/compose.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ which was derived from [Docker Compose file version 3 specification](https://doc
- `services.<SERVICE>.deploy.resources.reservations`
- `services.<SERVICE>.deploy.placement`
- `services.<SERVICE>.deploy.endpoint_mode`
- `services.<SERVICE>.healthcheck.start_interval`
- `services.<SERVICE>.stop_grace_period`
- `services.<SERVICE>.stop_signal`
- `configs.<CONFIG>.external`
Expand Down
15 changes: 12 additions & 3 deletions docs/healthchecks.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,11 @@ Health checks can be configured in multiple ways:
- `--health-timeout`: Maximum time to allow one check to run (default: 30s)
- `--health-retries`: Consecutive failures needed to report unhealthy (default: 3)
- `--health-start-period`: Start period for the container to initialize before starting health-retries countdown
- `--health-start-interval`: Time between running the check during the start period (default: 5s)
- `--no-healthcheck`: Disable any container-specified HEALTHCHECK

2. At image build time using HEALTHCHECK in a Dockerfile

**Note:** The `--health-start-interval` option is currently not supported by nerdctl.

## Configuration Priority

When a container is created, nerdctl determines the health check configuration based on this priority:
Expand Down Expand Up @@ -90,7 +89,17 @@ nerdctl run -d --name app \
myapp
```

3. Disable health checks:
3. Health check that probes more frequently while starting up:
```bash
nerdctl run -d --name app \
--health-cmd="./health-check.sh" \
--health-interval=30s \
--health-start-period=60s \
--health-start-interval=5s \
myapp
```

4. Disable health checks:
```bash
nerdctl run --no-healthcheck myapp
```
13 changes: 7 additions & 6 deletions pkg/api/types/container_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,13 @@ type ContainerCreateOptions struct {
ImagePullOpt ImagePullOptions

// Healthcheck related fields
HealthCmd string
HealthInterval time.Duration
HealthTimeout time.Duration
HealthRetries int
HealthStartPeriod time.Duration
NoHealthcheck bool
HealthCmd string
HealthInterval time.Duration
HealthTimeout time.Duration
HealthRetries int
HealthStartPeriod time.Duration
HealthStartInterval time.Duration
NoHealthcheck bool

// UserNS name for user namespace mapping of container
UserNS string
Expand Down
3 changes: 3 additions & 0 deletions pkg/cmd/container/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,9 @@ func withHealthcheck(options types.ContainerCreateOptions, ensuredImage *imgutil
if options.HealthStartPeriod != 0 {
hc.StartPeriod = options.HealthStartPeriod
}
if options.HealthStartInterval != 0 {
hc.StartInterval = options.HealthStartInterval
}

// Apply defaults for any unset values, but only if we have a healthcheck configured
if len(hc.Test) > 0 && hc.Test[0] != "NONE" {
Expand Down
1 change: 1 addition & 0 deletions pkg/cmd/container/health_check.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ func HealthCheck(ctx context.Context, client *containerd.Client, container conta
hcConfig.Interval = timeoutWithDefault(hcConfig.Interval, healthcheck.DefaultProbeInterval)
hcConfig.Timeout = timeoutWithDefault(hcConfig.Timeout, healthcheck.DefaultProbeTimeout)
hcConfig.StartPeriod = timeoutWithDefault(hcConfig.StartPeriod, healthcheck.DefaultStartPeriod)
hcConfig.StartInterval = timeoutWithDefault(hcConfig.StartInterval, healthcheck.DefaultProbeStartInterval)
if hcConfig.Retries == 0 {
hcConfig.Retries = healthcheck.DefaultProbeRetries
}
Expand Down
5 changes: 4 additions & 1 deletion pkg/composer/serviceparser/serviceparser.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,9 +130,9 @@ func warnUnknownFields(svc types.ServiceConfig) {
"Interval",
"Retries",
"StartPeriod",
"StartInterval",
"Disable",
"Extensions",
// TODO: add support 'StartInterval'
); len(unknown) > 0 {
log.L.Warnf("Ignoring: service %s: healthcheck: %+v", svc.Name, unknown)
}
Expand Down Expand Up @@ -833,6 +833,9 @@ func newContainer(project *types.Project, parsed *Service, i int) (*Container, e
if hc.StartPeriod != nil {
c.RunArgs = append(c.RunArgs, fmt.Sprintf("--health-start-period=%s", time.Duration(*hc.StartPeriod).String()))
}
if hc.StartInterval != nil {
c.RunArgs = append(c.RunArgs, fmt.Sprintf("--health-start-interval=%s", time.Duration(*hc.StartInterval).String()))
}
}
}

Expand Down
2 changes: 2 additions & 0 deletions pkg/composer/serviceparser/serviceparser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -936,6 +936,7 @@ services:
timeout: 10s
retries: 3
start_period: 5s
start_interval: 2s
cmd_exec:
image: alpine:3.14
healthcheck:
Expand Down Expand Up @@ -965,6 +966,7 @@ services:
assert.Assert(t, in(c.RunArgs, "--health-timeout=10s"))
assert.Assert(t, in(c.RunArgs, "--health-retries=3"))
assert.Assert(t, in(c.RunArgs, "--health-start-period=5s"))
assert.Assert(t, in(c.RunArgs, "--health-start-interval=2s"))

c = getContainersFromService(t, project, "cmd_exec")[0]
assert.Assert(t, in(c.RunArgs, "--health-cmd=curl -f http://localhost"))
Expand Down
Loading
Loading