diff --git a/experiment.yml b/experiment.yml index eb2c434..2803fcd 100644 --- a/experiment.yml +++ b/experiment.yml @@ -42,15 +42,15 @@ experiments: storage: workloads: default: - image: docker.io/jimjuniorb/fio:0.1 + image: docker.io/xridge/fio:latest snapshotter: overlayfs other: - - image: docker.io/jimjuniorb/fio-urunc:0.5 + - image: docker.io/jimjuniorb/fio-urunc:0.8 runtime: urunc snapshotter: devmapper - - image: docker.io/jimjuniorb/fio-urunc:0.5 + - image: docker.io/jimjuniorb/fio-urunc:0.8 runtime: urunc snapshotter: overlayfs diff --git a/images/fio-urunc/bunnyfile b/images/fio-urunc/bunnyfile index 7ccf8ad..bb6f96d 100644 --- a/images/fio-urunc/bunnyfile +++ b/images/fio-urunc/bunnyfile @@ -7,12 +7,16 @@ platforms: architecture: x86 rootfs: - from: jimjuniorb/fio:0.1 + from: xridge/fio:latest type: raw + include: + - from: harbor.nbfc.io/nubificus/urunit:latest + source: /urunit + destination: /urunit kernel: from: local path: bzImage -entrypoint: ["/usr/bin/fio"] \ No newline at end of file +entrypoint: ["/urunit", "/usr/bin/fio"] \ No newline at end of file diff --git a/images/fio/Dockerfile b/images/fio/Dockerfile deleted file mode 100644 index 88180c8..0000000 --- a/images/fio/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM alpine:3.22 - -RUN apk add --no-cache fio \ - && mkdir -p /bench - -WORKDIR /bench - -ENTRYPOINT ["fio"] \ No newline at end of file diff --git a/internal/runtime/storage/adapter.go b/internal/runtime/storage/adapter.go index 986d6a6..d5bfe5e 100644 --- a/internal/runtime/storage/adapter.go +++ b/internal/runtime/storage/adapter.go @@ -6,72 +6,25 @@ import ( "encoding/json" "fmt" "log" + "os" + "os/exec" "strings" "time" "github.com/containerd/containerd" - "github.com/containerd/containerd/cio" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/oci" harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" ) var processArgs = []string{ "--name=test-runtime", - "--directory=/bench", - "--rw=randrw", + "--rw=write", "--bs=4k", - "--size=512M", - "--direct=1", - "--time_based", + "--size=256M", "--runtime=30", - "--group_reporting", + "--time_based", "--output-format=json", } -func getOrPullImage( - ctx context.Context, - client *containerd.Client, - ref string, - snapshotter string, -) (containerd.Image, error) { - image, err := client.GetImage(ctx, ref) - if err == nil { - if err := image.Unpack(ctx, snapshotter); err != nil && - !errdefs.IsAlreadyExists(err) { - return nil, fmt.Errorf( - "unpack image %q using snapshotter %q: %w", - ref, - snapshotter, - err, - ) - } - - return image, nil - } - - if !errdefs.IsNotFound(err) { - return nil, fmt.Errorf("get image %q: %w", ref, err) - } - - image, err = client.Pull( - ctx, - ref, - containerd.WithPullUnpack, - containerd.WithPullSnapshotter(snapshotter), - ) - if err != nil { - return nil, fmt.Errorf( - "pull image %q using snapshotter %q: %w", - ref, - snapshotter, - err, - ) - } - - return image, nil -} - type Adapter struct { ContainerdClient *containerd.Client ContainerdNamespace *context.Context @@ -93,153 +46,72 @@ func (a *Adapter) ExperimentName() string { return "storage" } -func (a *Adapter) Prepare(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - - if tc.Trial.RuntimeName == "runsc" { - return a.CLIPrepare(tc) - } - - log.Printf("STORAGE: Preparing trial %s with runtime %s and handler %s and snapshotter %s and image %s", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, tc.Trial.Snapshotter, tc.Trial.Image) - startedAt := time.Now() - - // image - image, err := getOrPullImage(*a.ContainerdNamespace, a.ContainerdClient, tc.Trial.Image, tc.Trial.Snapshotter) - if err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("get or pull image %q: %w", tc.Trial.Image, err) - } - // create container metadata - container, err := a.ContainerdClient.NewContainer( - *a.ContainerdNamespace, - tc.Trial.ID, - containerd.WithImage(image), - containerd.WithSnapshotter(tc.Trial.Snapshotter), - containerd.WithNewSnapshot(tc.Trial.ID+"-snapshot", image), - containerd.WithRuntime(tc.Trial.RuntimeHandler, nil), - containerd.WithNewSpec( - oci.WithImageConfigArgs(image, processArgs), - ), - ) - if err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("create container %q: %w", tc.Trial.ID, err) - } - - a.Container = container - - finishedAt := time.Now() - - log.Printf("STORAGE: Finished preparing trial %s with runtime %s and handler %s and snapshotter %s and image %s", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, tc.Trial.Snapshotter, tc.Trial.Image) +func (a *Adapter) Prepare( + ctx context.Context, + tc harnessruntime.TrialContext, +) (harnessruntime.StageResult, error) { - return harnessruntime.StageResult{ - Stage: harnessruntime.StagePrepare, - StartedAt: startedAt, - FinishedAt: finishedAt, - Duration: finishedAt.Sub(startedAt), - Description: fmt.Sprintf( - "%s: trial=%s runtime=%s handler=%s image=%s", - "Setup container Metadata", - tc.Trial.ID, - tc.Trial.RuntimeName, - tc.Trial.RuntimeHandler, - tc.Trial.Image, - ), - Data: map[string]interface{}{ - "start": startedAt, - "end": finishedAt, - "latency": finishedAt.Sub(startedAt), - }, - }, nil + return fakeStage(context.Background(), harnessruntime.StagePrepare, "would prepare the trial", tc) } -func (a *Adapter) CreateTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - - if tc.Trial.RuntimeName == "runsc" { - return a.CLICreateTask(tc) - } - - log.Printf("STORAGE: Creating task for trial %s with runtime %s and handler %s and snapshotter %s and image %s", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, tc.Trial.Snapshotter, tc.Trial.Image) - var latency time.Duration - - startedAt := time.Now() - - task, err := a.Container.NewTask(*a.ContainerdNamespace, cio.NewCreator( - cio.WithStreams(nil, &a.StdoutBuffer, &a.StderrBuffer), - )) - if err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("create task %q: %w", tc.Trial.ID, err) - } - a.Task = task - - exitCh, err := task.Wait(*a.ContainerdNamespace) - if err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("wait on task %q: %w", tc.Trial.ID, err) - } - - a.TaskExitCh = exitCh - - endTime := time.Now() - - latency = endTime.Sub(startedAt) - log.Printf("STORAGE: Finished creating task for trial %s with runtime %s and handler %s and snapshotter %s and image %s, latency: %v", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, tc.Trial.Snapshotter, tc.Trial.Image, latency) - - return harnessruntime.StageResult{ - Stage: harnessruntime.StageCreate, - StartedAt: startedAt, - FinishedAt: endTime, - Duration: latency, - Description: fmt.Sprintf( - "%s: trial=%s runtime=%s handler=%s image=%s", - "Setup container Metadata", - tc.Trial.ID, - tc.Trial.RuntimeName, - tc.Trial.RuntimeHandler, - tc.Trial.Image, - ), - Data: map[string]interface{}{ - "start": startedAt, - "end": endTime, - "latency": latency, - "latency_ms": float64(latency.Microseconds()) / 1000, - }, - }, nil +func (a *Adapter) CreateTask( + ctx context.Context, + tc harnessruntime.TrialContext, +) (harnessruntime.StageResult, error) { + return fakeStage(context.Background(), harnessruntime.StageCreate, "would create the task", tc) } func (a *Adapter) StartTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - - if tc.Trial.RuntimeName == "runsc" { - return a.CLIStartTask(tc) - } - startedAt := time.Now() - if err := a.Task.Start(*a.ContainerdNamespace); err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("start task %q: %w", tc.Trial.ID, err) + cmdArgs := []string{ + "run", + "--rm", + "-it", + fmt.Sprintf("--runtime=%s", tc.Trial.RuntimeHandler), + tc.Trial.Image, } + cmdArgs = append(cmdArgs, processArgs...) + cmdNerdctl := exec.Command("nerdctl", cmdArgs...) - finishedAt := time.Now() + var stdout bytes.Buffer + var stderr bytes.Buffer - var exitCode uint32 + cmdNerdctl.Stdin = os.Stdin + cmdNerdctl.Stdout = &stdout + cmdNerdctl.Stderr = &stderr - // wait for the task to exit and also get exit code - select { - case <-ctx.Done(): - return harnessruntime.StageResult{}, ctx.Err() - case status := <-a.TaskExitCh: - exitCodee, _, err := status.Result() - if err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("read task exit status: %w", err) - } - - exitCode = exitCodee + log.Printf("Running nerdctl command: %s\n", cmdNerdctl.String()) + if err := cmdNerdctl.Run(); err != nil { + return harnessruntime.StageResult{}, fmt.Errorf( + "start task with nerdctl: %w\nstdout: %s\nstderr: %s", + err, + stdout.String(), + stderr.String(), + ) } - fioJSON, err := extractFIOJSON(a.StdoutBuffer.String()) + finishedAt := time.Now() + + fioJSON, err := extractFIOJSON(stdout.String()) if err != nil { return harnessruntime.StageResult{}, fmt.Errorf( "extract fio output: %w; raw stdout=%q", err, - a.StdoutBuffer.String(), + stdout.String(), + ) + } + + // Validate that fio actually returned valid JSON. + var fioResult map[string]interface{} + if err := json.Unmarshal([]byte(fioJSON), &fioResult); err != nil { + return harnessruntime.StageResult{}, fmt.Errorf( + "fio returned invalid JSON: %w\nstdout: %s\nstderr: %s", + err, + stdout.String(), + stderr.String(), ) } @@ -250,78 +122,38 @@ func (a *Adapter) StartTask(ctx context.Context, tc harnessruntime.TrialContext) Duration: finishedAt.Sub(startedAt), Description: fmt.Sprintf( "%s: trial=%s runtime=%s handler=%s image=%s", - "Start container task", + "Start task", tc.Trial.ID, tc.Trial.RuntimeName, tc.Trial.RuntimeHandler, tc.Trial.Image, ), - // since the task has exited, we can include the exit code in the data, also incluse the stdout and stderr buffers Data: map[string]interface{}{ "start": startedAt, "end": finishedAt, "latency": finishedAt.Sub(startedAt), - "latency_ms": float64(finishedAt.Sub(startedAt).Microseconds()) / 1000, - "stdout": string(fioJSON), - "stderr": a.StderrBuffer.String(), - "exit_code": exitCode, + "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), + "stdout": fioJSON, + "stderr": stderr.String(), }, }, nil - -} - -func (a *Adapter) WaitReady(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - return fakeStage(ctx, harnessruntime.StageWaitReady, "would wait for READY event or health endpoint", tc) } func (a *Adapter) Stop(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - return fakeStage(ctx, harnessruntime.StageCleanup, "would audit and clean runtime leftovers", tc) + return fakeStage(context.Background(), harnessruntime.StageStop, "would stop the task", tc) } func (a *Adapter) DeleteTask(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - if tc.Trial.RuntimeName == "runsc" { - return a.CLIDeleteTask(tc) - } - - startedAt := time.Now() - - if _, err := a.Task.Delete(*a.ContainerdNamespace); err != nil { - return harnessruntime.StageResult{}, fmt.Errorf("delete task %q: %w", tc.Trial.ID, err) - } - - finishedAt := time.Now() - - return harnessruntime.StageResult{ - Stage: harnessruntime.StageDelete, - StartedAt: startedAt, - FinishedAt: finishedAt, - Duration: finishedAt.Sub(startedAt), - Description: fmt.Sprintf( - "%s: trial=%s runtime=%s handler=%s image=%s", - "Delete container task", - tc.Trial.ID, - tc.Trial.RuntimeName, - tc.Trial.RuntimeHandler, - tc.Trial.Image, - ), - Data: map[string]interface{}{ - "start": startedAt, - "end": finishedAt, - "latency": finishedAt.Sub(startedAt), - "latency_ms": float64(finishedAt.Sub(startedAt).Microseconds()) / 1000, - }, - }, nil + return fakeStage(context.Background(), harnessruntime.StageDelete, "would delete the task", tc) } func (a *Adapter) Cleanup(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - if tc.Trial.RuntimeName == "runsc" { - return a.CLICleanupTask(tc) - } - a.Container.Delete( - *a.ContainerdNamespace, - containerd.WithSnapshotCleanup, - ) - return fakeStage(ctx, harnessruntime.StageCleanup, "would audit and clean runtime leftovers", tc) + // delete the bundle directory + return fakeStage(context.Background(), harnessruntime.StageCleanup, "would audit and clean runtime leftovers", tc) +} + +func (a *Adapter) WaitReady(ctx context.Context, tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { + return fakeStage(ctx, harnessruntime.StageWaitReady, "would wait for READY event or health endpoint", tc) } func fakeStage( @@ -356,18 +188,18 @@ func fakeStage( }, nil } -func extractFIOJSON(stdout string) ([]byte, error) { +func extractFIOJSON(stdout string) (string, error) { // SeaBIOS and boot output occur before the JSON. jsonStart := strings.IndexByte(stdout, '{') if jsonStart == -1 { - return nil, fmt.Errorf("fio JSON start not found in stdout") + return "", fmt.Errorf("fio JSON start not found in stdout") } decoder := json.NewDecoder(strings.NewReader(stdout[jsonStart:])) var raw json.RawMessage if err := decoder.Decode(&raw); err != nil { - return nil, fmt.Errorf("decode fio JSON: %w", err) + return "", fmt.Errorf("decode fio JSON: %w", err) } // Confirm that what we extracted is actually fio output. @@ -380,24 +212,24 @@ func extractFIOJSON(stdout string) ([]byte, error) { } if err := json.Unmarshal(raw, &header); err != nil { - return nil, fmt.Errorf("validate fio JSON: %w", err) + return "", fmt.Errorf("validate fio JSON: %w", err) } if header.FIOVersion == "" { - return nil, fmt.Errorf("extracted JSON is not fio output: missing fio version") + return "", fmt.Errorf("extracted JSON is not fio output: missing fio version") } if len(header.Jobs) == 0 { - return nil, fmt.Errorf("fio JSON contains no jobs") + return "", fmt.Errorf("fio JSON contains no jobs") } // Return a clean, consistently formatted JSON byte slice. var cleaned bytes.Buffer if err := json.Indent(&cleaned, raw, "", " "); err != nil { - return nil, fmt.Errorf("format fio JSON: %w", err) + return "", fmt.Errorf("format fio JSON: %w", err) } - return cleaned.Bytes(), nil + return cleaned.String(), nil } func (a *Adapter) GenerateResult(ctx context.Context, tc harnessruntime.TrialContext, results []harnessruntime.StageResult) (any, error) { diff --git a/internal/runtime/storage/cli_adaptor.go b/internal/runtime/storage/cli_adaptor.go deleted file mode 100644 index 2b96f9a..0000000 --- a/internal/runtime/storage/cli_adaptor.go +++ /dev/null @@ -1,111 +0,0 @@ -package storage - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "log" - "os" - "os/exec" - "time" - - harnessruntime "github.com/urunc-dev/evaluation_suite/internal/runtime" -) - -func (a *Adapter) CLIPrepare( - tc harnessruntime.TrialContext, -) (harnessruntime.StageResult, error) { - - return fakeStage(context.Background(), harnessruntime.StagePrepare, "would prepare the trial", tc) -} - -func (a *Adapter) CLICreateTask( - tc harnessruntime.TrialContext, -) (harnessruntime.StageResult, error) { - - return fakeStage(context.Background(), harnessruntime.StageCreate, "would create the task", tc) -} - -func (a *Adapter) CLIStartTask(tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - startedAt := time.Now() - - cmdArgs := []string{ - "run", - "--rm", - "-it", - fmt.Sprintf("--runtime=%s", tc.Trial.RuntimeHandler), - tc.Trial.Image, - } - cmdArgs = append(cmdArgs, processArgs...) - cmdNerdctl := exec.Command("nerdctl", cmdArgs...) - - var stdout bytes.Buffer - var stderr bytes.Buffer - - cmdNerdctl.Stdin = os.Stdin - cmdNerdctl.Stdout = &stdout - cmdNerdctl.Stderr = &stderr - - log.Printf("Running nerdctl command: %s\n", cmdNerdctl.String()) - - if err := cmdNerdctl.Run(); err != nil { - return harnessruntime.StageResult{}, fmt.Errorf( - "start task with nerdctl: %w\nstdout: %s\nstderr: %s", - err, - stdout.String(), - stderr.String(), - ) - } - - finishedAt := time.Now() - - fioJSON := stdout.String() - - // Validate that fio actually returned valid JSON. - var fioResult map[string]interface{} - if err := json.Unmarshal(stdout.Bytes(), &fioResult); err != nil { - return harnessruntime.StageResult{}, fmt.Errorf( - "fio returned invalid JSON: %w\nstdout: %s\nstderr: %s", - err, - stdout.String(), - stderr.String(), - ) - } - - return harnessruntime.StageResult{ - Stage: harnessruntime.StageStart, - StartedAt: startedAt, - FinishedAt: finishedAt, - Duration: finishedAt.Sub(startedAt), - Description: fmt.Sprintf( - "%s: trial=%s runtime=%s handler=%s image=%s", - "Start task", - tc.Trial.ID, - tc.Trial.RuntimeName, - tc.Trial.RuntimeHandler, - tc.Trial.Image, - ), - Data: map[string]interface{}{ - "start": startedAt, - "end": finishedAt, - "latency": finishedAt.Sub(startedAt), - "latency_ms": finishedAt.Sub(startedAt).Milliseconds(), - "stdout": fioJSON, - "stderr": stderr.String(), - }, - }, nil -} - -func (a *Adapter) CLIStopTask(tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - return fakeStage(context.Background(), harnessruntime.StageStop, "would stop the task", tc) -} - -func (a *Adapter) CLIDeleteTask(tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - return fakeStage(context.Background(), harnessruntime.StageDelete, "would delete the task", tc) -} - -func (a *Adapter) CLICleanupTask(tc harnessruntime.TrialContext) (harnessruntime.StageResult, error) { - // delete the bundle directory - return fakeStage(context.Background(), harnessruntime.StageCleanup, "would audit and clean runtime leftovers", tc) -}