diff --git a/fern/versions/latest.yml b/fern/versions/latest.yml index 88c14721c..02e254693 100644 --- a/fern/versions/latest.yml +++ b/fern/versions/latest.yml @@ -81,8 +81,28 @@ navigation: path: ./latest/pages/notebooks/6-editing-images-with-image-context.mdx - section: Slurm contents: + - page: Overview + path: ./latest/pages/slurm/overview.mdx + - page: Getting Started + path: ./latest/pages/slurm/getting-started.mdx + - page: Profiles and Clusters + path: ./latest/pages/slurm/profiles.mdx + - page: Images + path: ./latest/pages/slurm/images.mdx + - page: Run Configuration + path: ./latest/pages/slurm/run-configuration.mdx + - page: Run Operations + path: ./latest/pages/slurm/operations.mdx + - page: Retry and Collection + path: ./latest/pages/slurm/retry-and-collection.mdx - page: Benchmarks path: ./latest/pages/slurm/benchmarks.mdx + - page: Dependencies and Tools + path: ./latest/pages/slurm/dependencies-and-tools.mdx + - page: Troubleshooting + path: ./latest/pages/slurm/troubleshooting.mdx + - page: CLI Reference + path: ./latest/pages/slurm/cli-reference.mdx - section: Recipes contents: - page: Recipe Cards diff --git a/fern/versions/latest/pages/slurm/benchmarks.mdx b/fern/versions/latest/pages/slurm/benchmarks.mdx index f1161462d..059f46e6a 100644 --- a/fern/versions/latest/pages/slurm/benchmarks.mdx +++ b/fern/versions/latest/pages/slurm/benchmarks.mdx @@ -2,6 +2,8 @@ Use a benchmark to compare concurrency and deployment topology while keeping each case as an ordinary Data Designer Slurm run. No benchmark controller stays resident after submission. +Complete [Getting Started](/slurm/getting-started) first so the profile, images, and base run are ready. + Install the Slurm extension: ```bash @@ -22,12 +24,12 @@ concurrency_values: - 32 - 64 deployment_cases: - - name: two-independent-replicas + - name: single-node-groups deployments: generator: nodes: 2 nodes_per_replica: 1 - - name: one-two-node-replica + - name: two-node-group deployments: generator: nodes: 2 @@ -42,7 +44,9 @@ analysis: target_runtime: 4h ``` -The compiler expands the authored order deterministically. For this example, the first cases are `two-independent-replicas-c32` and `one-two-node-replica-c32`. An adaptive record policy uses `ceil(concurrency * records_per_concurrency)`, bounded by `base_records` and `max_records`. +The compiler expands the authored order deterministically. For this example, the first cases are `single-node-groups-c32` and `two-node-group-c32`. An adaptive record policy uses `ceil(concurrency * records_per_concurrency)`, bounded by `base_records` and `max_records`. + +`nodes_per_replica` controls the number of nodes in each pipeline-parallel group. The resolved replica count is `(nodes / nodes_per_replica) * (gpus_per_node / tensor_parallel)`, so it also depends on the selected profile and the base run's deployment topology. ## Run @@ -92,3 +96,5 @@ service = create_slurm_benchmark_service( manifest = service.run(config, source_root=benchmark_file.parent) report = service.analyze(manifest.benchmark_id, refresh_state=True) ``` + +For child failure recovery and output collection, use [Retry and collection](/slurm/retry-and-collection). diff --git a/fern/versions/latest/pages/slurm/cli-reference.mdx b/fern/versions/latest/pages/slurm/cli-reference.mdx new file mode 100644 index 000000000..8566d34fa --- /dev/null +++ b/fern/versions/latest/pages/slurm/cli-reference.mdx @@ -0,0 +1,42 @@ +# Slurm CLI reference + +Install the extension with `python -m pip install "data-designer[slurm]"`, verify it with `python -m pip show data-designer-slurm`, then use `data-designer slurm COMMAND --help` for the authoritative option list. + +All commands accept `--profile-file PATH` and `--cluster NAME` where cluster selection is required. + +| Command | Purpose | +| --- | --- | +| `profile init` | Create a restrictive starter profile catalog without overwriting. | +| `profile validate` | Validate catalog selection, shared workspace access, and scheduler facts. | +| `image add` | Import or inspect an image through a CPU Slurm job and publish an alias. | +| `image ls` | List aliases in the selected profile workspace. | +| `image info NAME` | Show one registered alias and immutable image facts. | +| `image rm NAME` | Remove an alias without deleting its SQSH artifact. | +| `execute RUN_FILE` | Resolve and submit a run. Add `--dry-run` to render without submission. | +| `status RUN_ID` | Reconcile scheduler observations with durable managed state. | +| `cancel RUN_ID` | Cancel active jobs owned by a managed run. | +| `retry RUN_OR_JOB_ID` | Preview or submit a sparse retry. Repeat `--task-id` to select shards. | +| `merge` | Submit a zero-GPU winner collection job for a managed run directory. | +| `benchmark run BENCHMARK_FILE` | Expand benchmark cases into ordinary child runs and submit them. | +| `benchmark analyze BENCHMARK` | Write a report from persisted child state. | + +## Common options + +| Option | Commands | Meaning | +| --- | --- | --- | +| `--profile-file PATH` | Most commands | Use a catalog other than the default or environment-selected file. | +| `--cluster NAME` | Most commands | Select one catalog entry explicitly. | +| `--dry-run` | `execute`, `retry` | Perform resolution and render a batch script without submission. | +| `--force` | `retry`, `benchmark run` | Skip retry confirmation or resume a safely recoverable benchmark submission. It does not overwrite immutable evidence. | +| `--refresh-state` | `benchmark analyze` | Reconcile each child with the scheduler before analysis. | +| `--fail-if-incomplete` | `benchmark analyze` | Persist the report, then return a conflict if any case is incomplete. | + +## Output and errors + +Success is one canonical JSON value on stdout. Failure is one object on stderr: + +```json +{"error":{"code":"invalid_request","message":"...","operation":"execute"}} +``` + +Exit codes are `1` for internal errors, `2` for invalid requests, `3` for missing managed resources, `4` for conflicts, and `5` for unavailable external systems. Error messages are bounded and sanitized; inspect durable logs for operational detail. diff --git a/fern/versions/latest/pages/slurm/dependencies-and-tools.mdx b/fern/versions/latest/pages/slurm/dependencies-and-tools.mdx new file mode 100644 index 000000000..25f2affcd --- /dev/null +++ b/fern/versions/latest/pages/slurm/dependencies-and-tools.mdx @@ -0,0 +1,115 @@ +# Dependencies, plugins, MCP, and secrets + +The client image provides Data Designer and its base environment. A run may add immutable pure Python wheels as an allocation-local overlay. The overlay is reconstructed for each allocation and is not written back into the image. + +## Add package requirements + +```yaml +client: + image: + name: dd-client + dependencies: + requirements: + - example-plugin==1.2.3 +``` + +The submit-side resolver accepts package requirements or direct HTTPS wheel URLs with a SHA-256 fragment. It rejects editable installs, source directories, VCS references, markers, duplicate normalized names, and non-wheel URLs. The resolved lock and wheels are persisted with the run. + +Do not use the overlay to replace `data-designer`, `data-designer-config`, `data-designer-engine`, or `data-designer-slurm`. Build a new client image when these packages need to change. + +## Reuse a dependency lock + +Use exactly one of `requirements` or `lock_file`: + +```yaml +client: + image: + name: dd-client + dependencies: + requirements: null + lock_file: inputs/dependency-lock.json +``` + +The lock must target the registered client image inventory and reference available wheel artifacts. + +`client.dependencies.index_credentials` reserves named secret references for the allocation runtime. Submit-side inline requirement resolution does not use them and always targets the public index. + +Inline requirements resolve from the public Python package index with ambient pip configuration disabled. For private or unpublished plugins, bake the package into the client image, use an immutable HTTPS wheel URL that does not embed credentials, or supply a previously resolved compatible lock and wheel artifacts. + +## Reference secrets + +Secrets are environment references, never literal values in persisted YAML: + +```yaml +deployments: + - model_alias: generator + model: example/generator + server: + type: vllm + image: + name: vllm + environment: + MODEL_ACCESS_TOKEN: + type: secret + environment: MODEL_ACCESS_TOKEN +``` + +Literal non-secret settings use an explicit literal binding: + +```yaml + environment: + VLLM_LOGGING_LEVEL: + type: literal + value: INFO + MODEL_ACCESS_TOKEN: + type: secret + environment: MODEL_ACCESS_TOKEN +``` + +Export referenced variables in the trusted submit environment before running `execute`. Values are forwarded only at the required runtime boundary and are not serialized into run contracts or logs. Secret-shaped environment names cannot use literal bindings. + +## Use plugins + +Install plugin packages into the client image or declare a compatible pure-wheel dependency. The serialized builder config can then reference the plugin column, seed reader, or processor. See the [plugin overview](/plugins/overview) for Data Designer plugin contracts. + +## Configure MCP providers + +Remote providers support SSE and streamable HTTP: + +```yaml +invocation: + num_records: 100 + dataset_name: tool-use + mcp_providers: + - provider_type: streamable_http + name: search + endpoint: https://tools.example.test/mcp + api_key: + type: secret + environment: MCP_API_KEY +``` + +Endpoints cannot contain credentials, query parameters, or fragments. + +Local stdio providers run inside the client allocation: + +```yaml +invocation: + num_records: 100 + dataset_name: tool-use + mcp_providers: + - provider_type: stdio + name: local-tool + command: example-mcp-server + args: + - --mode + - read-only + environment: + TOOL_API_KEY: + type: secret + environment: TOOL_API_KEY +``` + +The command is one executable token and arguments are passed without a shell. The client image or dependency overlay must provide the executable. See [Tool Use and MCP](/concepts/tool-use-mcp/overview) for builder-side tool configuration. + +Code-sandbox sidecars are not supported by the v1 Slurm extension. Use an MCP service with an appropriate isolation boundary or run the workload through a separately managed sandbox system. diff --git a/fern/versions/latest/pages/slurm/getting-started.mdx b/fern/versions/latest/pages/slurm/getting-started.mdx new file mode 100644 index 000000000..f5cd08449 --- /dev/null +++ b/fern/versions/latest/pages/slurm/getting-started.mdx @@ -0,0 +1,205 @@ +# Slurm Getting Started + +This flow starts from a clean Python environment in a site-approved Slurm submit environment and submits one small Data Designer run. Replace the example account, partitions, image sources, and model with values available on your cluster. + +Before starting, confirm the [Slurm prerequisites and host responsibilities](/slurm/overview). + +Obtain these site-specific values from your Slurm or platform administrator before continuing: + +- A Python 3.10 or later executable +- An account, GPU partition, CPU image-build partition, and shared workspace path +- A digest-qualified client OCI image or absolute SQSH path containing the same Data Designer package versions as the submit environment +- A digest-qualified serving OCI image or absolute SQSH path containing a supported vLLM installation +- A model available to the serving image and compute nodes + +The image references and `example/generator` model below are syntax examples. They are intentionally not runnable values. + +## 1. Create and activate an environment + +Use a shared path instead of `$HOME` when required by site policy. If software installation is not allowed on login nodes, run these steps from a dedicated editor node or a short CPU allocation approved by your administrator. + +```bash +export DD_SLURM_WORKSPACE=/workspace/alice/data-designer-slurm +export DATA_DESIGNER_SLURM_PROFILE_FILE="$DD_SLURM_WORKSPACE/config/slurm-profiles.yml" +export DD_PYTHON="${DD_PYTHON:-python3}" + +mkdir -p "$DD_SLURM_WORKSPACE/config" "$DD_SLURM_WORKSPACE/envs" +"$DD_PYTHON" -c 'import sys; print(sys.version); raise SystemExit(sys.version_info < (3, 10))' +"$DD_PYTHON" -m venv "$DD_SLURM_WORKSPACE/envs/control" +source "$DD_SLURM_WORKSPACE/envs/control/bin/activate" +python -m pip install --upgrade pip +``` + +If the version check fails, load a site-provided Python module or set `DD_PYTHON` to a supported executable such as `python3.10`, then rerun the block. + +## 2. Install Data Designer with Slurm support + +```bash +python -m pip install "data-designer[slurm]" +python -m pip show data-designer-slurm +``` + +The base `data-designer` wheel does not install or import Slurm support. The extra installs the same-version `data-designer-slurm` wheel and exposes the lazy `data-designer slurm` command group. Do not continue if `pip show` cannot find the extension. + +## 3. Discover the CLI + +```bash +data-designer --help +data-designer slurm --help +``` + +If the Slurm command group is absent, see [Troubleshooting](/slurm/troubleshooting#the-slurm-command-group-is-missing). + +## 4. Create a profile + +Create the durable shared workspace first, then initialize an explicit profile: + +```bash +data-designer slurm profile init \ + --profile-file "$DATA_DESIGNER_SLURM_PROFILE_FILE" \ + --cluster primary \ + --workspace-root "$DD_SLURM_WORKSPACE" \ + --image-build-partition cpu \ + --account research \ + --partition gpu \ + --host-pattern 'login*.example.test' +``` + +The command writes the profile with restrictive permissions and never overwrites an existing file. Before submitting work, review every generated site value. In particular, confirm the account, partitions, hostname pattern, workspace, GPU request mode, `gpus_per_node`, and the image-build CPU, memory, and time limit. The starter sets `gpus_per_node: auto`, which requests the GPU count reported by eligible nodes. For this one-GPU walkthrough on a shared GRES cluster, set `gpus_per_node: 1` when your site permits partial-node allocations. If Slurm commands are not on the batch-job `PATH`, also set `scheduler.bin_path` to their absolute directory. The starter image-build time limit is four hours and may exceed site policy. Edit it before `image add` when necessary. + +## 5. Validate the profile + +```bash +data-designer slurm profile validate --cluster primary +``` + +Validation checks strict profile loading, cluster selection, workspace access, and scheduler facts. Keep `--cluster primary` explicit when running from a compute host whose hostname does not match the login-host pattern. See [Profiles and clusters](/slurm/profiles-and-clusters) for non-default files and multiple clusters. + +## 6. Register a client image + +Use a digest-qualified OCI reference: + +```bash +data-designer slurm image add \ + registry.example.test/data-designer/client@sha256:0000000000000000000000000000000000000000000000000000000000000000 \ + --kind client \ + --name dd-client \ + --cluster primary +``` + +Replace the entire source with the digest-qualified OCI reference or absolute SQSH path supplied for your site. See [Prepare compatible images](/slurm/images#prepare-compatible-images) before registering a client image. + +## 7. Register a serving image + +```bash +data-designer slurm image add \ + registry.example.test/inference/vllm@sha256:1111111111111111111111111111111111111111111111111111111111111111 \ + --kind serving \ + --name vllm \ + --cluster primary +``` + +Each `image add` command submits a CPU Slurm job, waits for it to finish, inspects the result, and only then publishes the alias. Keep the terminal open. You can also register an existing absolute `.sqsh` path. See [Images](/slurm/images). + +## 8. Create a Data Designer builder config + +Create `builder.yaml`: + +```yaml +data_designer: + model_configs: + - alias: generator + model: example/generator + provider: openai + columns: + - name: greeting + column_type: llm-text + prompt: Write one short greeting. + model_alias: generator +``` + +The model alias in this file must match a deployment in the Slurm run config. + +## 9. Create a Slurm run config + +Create `run.yaml` next to `builder.yaml`: + +```yaml +schema_version: 1 +name: greeting-run +builder: + source: builder.yaml +invocation: + num_records: 8 + dataset_name: greetings + model_concurrency: + generator: 8 +client: + image: + name: dd-client +deployments: + - model_alias: generator + served_model_name: example/generator + model: example/generator + server: + type: vllm + image: + name: vllm + startup_timeout: 15m + distributed_init_timeout: 10m + topology: + tensor_parallel: 1 +submission: + time_limit: 01:00:00 +output: + format: parquet + partitions: 1 +``` + +The repository also contains these checked examples under `packages/data-designer-slurm/examples`. `served_model_name` is the name sent by the client to the vLLM-compatible endpoint. It may differ from `model`, which can be an absolute model path visible only to the serving container. + +## 10. Dry-run the submission + +```bash +data-designer slurm execute run.yaml --dry-run --cluster primary +``` + +Dry-run resolves the selected profile, verifies registered image contents, loads the builder envelope, resolves client dependencies, and renders the batch script without calling `sbatch`. Full builder and plugin validation runs inside the prepared client environment on the compute node. Preview staging is temporary: dry-run does not initialize durable managed-run state and cannot be resumed or passed to `status`. + +## 11. Submit the run + +```bash +data-designer slurm execute run.yaml --cluster primary +``` + +The command prints one JSON object. Save its `run_id`; all lifecycle commands use the managed run ID rather than a raw Slurm job ID. + +## 12. Inspect or cancel the run + +```bash +data-designer slurm status --cluster primary +data-designer slurm cancel --cluster primary +``` + +Status reconciles active queue and accounting evidence with durable run state. Cancellation targets only jobs recorded for the managed run. + +## 13. Inspect results + +Each successful shard publishes a canonical winner at: + +```text +/workspace/alice/data-designer-slurm/runs//shards//winner.json +``` + +The winner's `candidate_manifest.path` points to the winning attempt's `output-manifest.json`. That manifest records `dataset_path` and the relative path, size, digest, and record count of every canonical output file. Follow this chain instead of recursively scanning for Parquet files, because failed or superseded attempts can retain non-canonical files. + +Use `status` to confirm every shard succeeded. To create one collected dataset at an explicit destination, submit a managed collection job: + +```bash +data-designer slurm merge \ + --input-path /workspace/alice/data-designer-slurm/runs/ \ + --output-path /workspace/alice/exports/greetings \ + --cluster primary +``` + +Continue with [Run operations](/slurm/run-operations), [Retry and collection](/slurm/retry-and-collection), [Scaling runs](/slurm/run-configuration#scale-runs), [Dependencies and tools](/slurm/dependencies-and-tools), or the [CLI reference](/slurm/cli-reference). diff --git a/fern/versions/latest/pages/slurm/images.mdx b/fern/versions/latest/pages/slurm/images.mdx new file mode 100644 index 000000000..0c6f921a0 --- /dev/null +++ b/fern/versions/latest/pages/slurm/images.mdx @@ -0,0 +1,80 @@ +# Slurm images + +Data Designer Slurm uses two image roles: + +- `client` images run Data Designer and installed plugins. +- `serving` images run vLLM model servers. + +Use separate aliases even if both roles originate from related images. Inspection verifies role-specific contents before an alias becomes available to planning. + +## Prepare compatible images + +Image construction is an operator prerequisite. Data Designer Slurm imports or inspects images, but does not build a client or serving environment from a Dockerfile. + +Obtain a digest-qualified OCI reference or absolute compute-visible SQSH path from your platform administrator, or publish one through your site's container build process. A client image must install the same release used by the submit environment: + +```bash +python -m pip install "data-designer[slurm]==" +``` + +For an unpublished build, install the aligned `data-designer`, `data-designer-config`, `data-designer-engine`, and `data-designer-slurm` wheels into the client image. Do not mix versions. A serving image must provide a supported vLLM executable and all runtime dependencies for the selected model. + +Before registration, confirm that the image source is accessible from the configured CPU image-build partition and that the model is accessible from the GPU partition. Keep registry credentials in the compute environment or Enroot configuration. + +## Import a digest-pinned OCI image + +The digest below demonstrates the required syntax and does not identify a real image. Replace it before running the command. + +```bash +data-designer slurm image add \ + registry.example.test/data-designer/client@sha256:0000000000000000000000000000000000000000000000000000000000000000 \ + --kind client \ + --name dd-client +``` + +OCI sources must use `repository@sha256:<64 lowercase hex characters>`. Schemes, embedded credentials, tags without a digest, query strings, and fragments are rejected. Registry credentials belong in the compute environment or Enroot configuration, not in the command or profile. + +The command stages package-owned inspection code, submits a CPU job using `image_build`, waits for terminal Slurm state, verifies the SQSH and inspection record, then atomically publishes the artifact and alias. The terminal stays attached while the job runs. + +## Register an existing SQSH + +```bash +data-designer slurm image add \ + /workspace/alice/images/client.sqsh \ + --kind client \ + --name dd-client +``` + +The absolute SQSH path must be visible on the image-build node and all later compute nodes. Existing images are inspected by a CPU job in place. + +## Inspect aliases + +```bash +data-designer slurm image ls +data-designer slurm image info dd-client +``` + +Aliases are cluster-profile scoped because each profile has its own workspace. Run configuration can reference an alias with `name` or a registered absolute path with `path`. + +## Replace or remove an alias + +An alias is immutable unless replacement is explicit: + +```bash +data-designer slurm image add /workspace/alice/images/client-v2.sqsh \ + --kind client \ + --name dd-client \ + --replace +``` + +Removing an alias does not delete its SQSH artifact: + +```bash +data-designer slurm image rm dd-client +``` + +Published OCI imports are content-addressed. Identical imports can share an artifact, while conflicting bytes or inspection facts fail closed. + +## Compatibility rules + +The client image inventory must contain the same versions of `data-designer`, `data-designer-config`, `data-designer-engine`, and `data-designer-slurm` used to submit the run. Dependency overlays may add pure Python wheels, but cannot replace these protected distributions. Serving images must expose a supported vLLM executable and version. diff --git a/fern/versions/latest/pages/slurm/operations.mdx b/fern/versions/latest/pages/slurm/operations.mdx new file mode 100644 index 000000000..283a22faa --- /dev/null +++ b/fern/versions/latest/pages/slurm/operations.mdx @@ -0,0 +1,71 @@ +# Slurm run operations + +The CLI prints canonical JSON to stdout. Errors are canonical JSON on stderr, so automation can consume the same commands as an operator. + +## Resolve without submitting + +```bash +data-designer slurm execute run.yaml --dry-run +``` + +Dry-run performs profile selection, image verification, builder-envelope loading, dependency resolution, plan compilation, runtime staging, and batch rendering. It does not call `sbatch`. Full builder and plugin validation runs later in the client environment. Because resolution is real, the selected workspace and registered images must already be available. Preview staging is temporary: dry-run does not create durable managed-run state, so its result cannot be resumed, inspected with `status`, or used as collection input. + +## Submit + +```bash +data-designer slurm execute run.yaml +``` + +A successful response includes `run_id`, `state`, `plan_sha256`, `shard_count`, and `job_id`. The package persists immutable inputs before submission and records scheduler identity after acceptance. Conflicting durable state is never overwritten. + +Use `--profile-file` and `--cluster` on any command when the defaults are not appropriate. + +## Inspect status + +```bash +data-designer slurm status +``` + +Status loads the managed run, queries active queue and accounting records for its known jobs, and reconciles those observations with persisted attempts, readiness, and shard winners. Accounting can lag after a job leaves the active queue; rerun status after the cluster accounting service catches up. + +The response includes every shard in planned order. Use the attempt lifecycle and readiness details to distinguish queue wait, model startup, active generation, publication, terminal failure, and success. + +## Cancel + +```bash +data-designer slurm cancel +``` + +Cancellation selects only active job IDs recorded for the managed run. It is safe to call again when no managed jobs remain active. + +## Durable artifacts + +Managed files are below `/runs/`: + +- Authored and resolved configuration, dependency lock, runtime bundle reference, and run manifest +- Slurm stdout and stderr +- Attempt and readiness records +- Candidate results, shard winners, collection state, and output + +Allocation-local scratch is deliberately absent after cleanup. The stable container path `/run/data-designer-slurm` is not a durable troubleshooting location. + +For each shard, `shards//winner.json` is the canonical pointer. Its `candidate_manifest.path` identifies the winning attempt's `output-manifest.json`, which records the canonical `dataset_path` and output files. Do not identify results by recursively scanning attempt directories: failed and superseded attempts may retain valid-looking data files. The `merge` command validates these winner chains before creating a run-level dataset at the requested destination. + +## Use the Python service API + +```python +from pathlib import Path + +from data_designer.slurm.config import load_run_config +from data_designer.slurm.services import create_slurm_run_service + +run_file = Path("run.yaml").resolve() +config = load_run_config(run_file) +service = create_slurm_run_service(cluster="primary") + +preview = service.execute(config, source_root=run_file.parent, dry_run=True) +submission = service.execute(config, source_root=run_file.parent) +status = service.status(submission.run_id) +``` + +Public service methods apply the same validation, persistence, error model, and output contracts as the CLI. diff --git a/fern/versions/latest/pages/slurm/overview.mdx b/fern/versions/latest/pages/slurm/overview.mdx new file mode 100644 index 000000000..4bd91d2f7 --- /dev/null +++ b/fern/versions/latest/pages/slurm/overview.mdx @@ -0,0 +1,39 @@ +# Data Designer on Slurm + +Data Designer Slurm runs Data Designer clients and vLLM model servers in batch allocations. The optional extension owns profile selection, image inspection, submission, status, retry, result collection, and benchmark workflows. + +Start with [Getting Started](/slurm/getting-started). Use [Dependencies and tools](/slurm/dependencies-and-tools) for plugins, MCP providers, and secrets, and keep the [CLI reference](/slurm/cli-reference) nearby for automation. + +## Prerequisites + +The submit environment needs: + +- Python 3.10 or later +- `sbatch`, `squeue`, `sacct`, `scancel`, `scontrol`, and `sinfo` +- Access to a shared workspace that is visible from the submit host and every eligible compute node + +Compute nodes need Enroot 3.5 or later for existing SQSH images. Digest-pinned OCI imports need Enroot 4.0 or later. The client image must contain the same Data Designer packages as the submit environment. Serving images must contain vLLM. + +Your Slurm administrator can confirm the account, GPU partition, CPU image-build partition, GPU request mode, and shared filesystem paths for your cluster. + +## Where each task runs + +| Persona | Responsibilities | +| --- | --- | +| Workstation | Author and version dataset and Slurm configuration. Copy files to a cluster-visible location before submission. | +| Site-approved submit environment | Install the CLI, select a profile, register images, submit work, inspect status, retry, cancel, and collect output. This may be a login host, editor node, or CPU allocation, depending on site policy. | +| Compute node | Import or inspect images, start vLLM and client containers, generate records, and write managed state and results. | + +Run Slurm commands from a site-approved submit environment with the cluster CLI and shared filesystem. Do not install packages or run substantive work on a login node when site policy prohibits it. A laptop without the cluster CLI and shared filesystem can author configuration, but cannot validate or submit it. + +## Storage model + +The configured `workspace_root` is durable and shared. It contains image metadata, immutable run inputs, Slurm logs, attempt state, winning results, collection state, and default output. + +Runtime bundle extraction, the client dependency overlay, Data Designer home and cache directories, and Enroot state use allocation-local scratch. The runtime mounts that scratch at `/run/data-designer-slurm` inside containers and removes it on success, failure, cancellation, and retry. Do not configure a mount that overlaps this path or expect transient scratch files to remain after a job. + +## Supported in v1 + +The public v1 surface supports named cluster profiles, client and serving image aliases, one or more vLLM deployments, independent replicas, multi-node replicas, Slurm arrays, bounded array concurrency, durable status, cancellation, sparse retry, winner-driven collection, dependencies, plugins, MCP providers, and benchmarks. + +Interactive allocations, code-sandbox sidecars, NeMo Run, SGLang, Dynamo, Kubernetes, Ray, and platform-specific orchestration are not part of the v1 Slurm extension. diff --git a/fern/versions/latest/pages/slurm/profiles.mdx b/fern/versions/latest/pages/slurm/profiles.mdx new file mode 100644 index 000000000..b7fc903b0 --- /dev/null +++ b/fern/versions/latest/pages/slurm/profiles.mdx @@ -0,0 +1,105 @@ +# Slurm profiles and clusters + +A profile catalog stores site facts separately from portable run intent. It selects one cluster and supplies scheduler defaults, workspace storage, image-build resources, GPU facts, and container mounts. + +## Locate the catalog + +Commands resolve the catalog in this order: + +1. `--profile-file PATH` +2. `DATA_DESIGNER_SLURM_PROFILE_FILE` +3. `~/.data-designer-slurm-profile.yml` + +Pass `--cluster NAME` to select an entry explicitly. Without it, hostname patterns are considered first, followed by `default_cluster` when there is no match. Duplicate patterns and ambiguous hostname matches are rejected. + +## Initialize a catalog + +```bash +mkdir -p /workspace/alice/config +data-designer slurm profile init \ + --profile-file /workspace/alice/config/slurm-profiles.yml \ + --workspace-root /workspace/alice/data-designer-slurm \ + --image-build-partition cpu \ + --cluster primary \ + --account research \ + --partition gpu \ + --host-pattern 'login*.example.test' +``` + +The starter sets `gpus_per_node: auto`, `gpu_request_mode: gres`, and CPU image-build resources of 2 CPUs, 8 GiB, and four hours. The parent directory of a custom `--profile-file` must already exist. Review the generated account, partitions, host patterns, workspace, GPU settings, and image-build resources against site policy before validation or image registration. + +## Configure multiple clusters + +```yaml +schema_version: 1 +default_cluster: primary +clusters: + primary: + schema_version: 1 + host_patterns: + - login*.example.test + scheduler: + account: research + partition: gpu + mem_per_gpu: 80G + bin_path: /opt/slurm/bin + gpus_per_node: 8 + workspace_root: /workspace/alice/data-designer-slurm + image_build: + partition: cpu + cpus_per_task: 2 + memory: 8G + time_limit: "04:00:00" + gpu_request_mode: gres + lab: + schema_version: 1 + host_patterns: + - lab-login*.example.test + scheduler: + partition: accelerated + gpus_per_node: 4 + workspace_root: /workspace/alice/data-designer-slurm-lab + image_build: + partition: general + cpus_per_task: 2 + memory: 8G + time_limit: "04:00:00" + gpu_request_mode: visible +``` + +Run a command against a non-default cluster: + +```bash +data-designer slurm profile validate --cluster lab +data-designer slurm image ls --cluster lab +data-designer slurm execute run.yaml --cluster lab +``` + +`visible` GPU mode asks Slurm for visible GPUs without GRES memory directives. It cannot be combined with `mem_per_gpu`. + +`gpus_per_node` is both the GPU count used for deployment topology and, in `gres` mode, the count requested from Slurm on each job node. `auto` queries eligible nodes and uses their reported count, so an eight-GPU partition produces `--gres=gpu:8`. Set an explicit smaller count, such as `1`, for partial-node allocations when the cluster permits them. Use `auto` only when requesting every GPU on each selected node is intended, and use an explicit count when eligible nodes are heterogeneous. + +Set `scheduler.bin_path` when Slurm executables are installed outside the generated batch job's default `PATH`. It must be one absolute directory and is prepended to `PATH`; do not provide a colon-separated list. + +## Add mounts + +Use `container_mounts` for absolute compute-visible paths that the client or model needs: + +```yaml +container_mounts: + - source: /workspace/alice/models + target: /models + read_only: true +``` + +Targets must be unique and must not overlap `/run/data-designer-slurm`, which is reserved for allocation-local scratch. + +## Validate before submission + +```bash +data-designer slurm profile validate \ + --profile-file /workspace/alice/config/slurm-profiles.yml \ + --cluster primary +``` + +Validation creates and removes a probe in `workspace_root`. With `gpus_per_node: auto`, it also queries eligible nodes and requires one consistent positive GPU count. diff --git a/fern/versions/latest/pages/slurm/retry-and-collection.mdx b/fern/versions/latest/pages/slurm/retry-and-collection.mdx new file mode 100644 index 000000000..7816bc75a --- /dev/null +++ b/fern/versions/latest/pages/slurm/retry-and-collection.mdx @@ -0,0 +1,56 @@ +# Retry and collection + +Retry and collection operate on persisted run state. They do not infer ownership from arbitrary Slurm jobs or directories. + +## Preview a retry + +```bash +data-designer slurm retry --dry-run +``` + +By default, retry selects failed or incomplete shards and resolves `--resume if_possible` to either `always` or `never` from persisted compatibility evidence. The response includes selected shard IDs, new attempt IDs, effective resume mode, and a batch script. + +Select specific zero-based array tasks when needed: + +```bash +data-designer slurm retry \ + --task-id 2 \ + --task-id 7 \ + --resume never \ + --dry-run +``` + +## Submit a retry + +```bash +data-designer slurm retry +``` + +Interactive use asks for confirmation. Add `--force` to skip that prompt in automation. Force does not replace immutable run metadata or discard scheduler evidence. + +You may pass a managed Slurm job ID instead of a run ID when the package can resolve it unambiguously. Prefer the run ID in scripts. + +Retries create new attempts. The original attempt records remain durable, and each shard publishes at most one winner. Status shows the complete attempt history. + +## Collect winners + +Default successful output is already managed beneath the run directory. Use `merge` to submit a zero-GPU CPU job that collects shard winners to an explicit compute-visible destination: + +```bash +data-designer slurm merge \ + --input-path /workspace/alice/data-designer-slurm/runs/ \ + --output-path /workspace/alice/exports/final-dataset +``` + +`--input-path` must be exactly a managed run directory under the selected profile workspace. The output path must be absolute, writable, compute-visible, and authorized by the profile mounts. + +If supplied, `--num-partitions` must equal the partition count persisted in the run: + +```bash +data-designer slurm merge \ + --input-path /workspace/alice/data-designer-slurm/runs/ \ + --output-path /workspace/alice/exports/final-dataset \ + --num-partitions 20 +``` + +Repeating collection returns or reuses managed state when safe. Conflicting destinations or partition counts are rejected. diff --git a/fern/versions/latest/pages/slurm/run-configuration.mdx b/fern/versions/latest/pages/slurm/run-configuration.mdx new file mode 100644 index 000000000..225d35aad --- /dev/null +++ b/fern/versions/latest/pages/slurm/run-configuration.mdx @@ -0,0 +1,186 @@ +# Slurm run configuration + +A run file is portable intent. Cluster-specific accounts, partitions, workspace paths, GPU facts, and image-build resources belong in a [profile](/slurm/profiles-and-clusters). + +## Required structure + +```yaml +schema_version: 1 +name: product-descriptions +builder: + source: builder.yaml +invocation: + num_records: 1000 + dataset_name: product-descriptions + model_concurrency: + generator: 64 +client: + cpus: 32 + image: + name: dd-client +deployments: + - model_alias: generator + served_model_name: example/generator + model: example/generator + server: + type: vllm + image: + name: vllm + resources: + nodes: 1 + topology: + tensor_parallel: 8 + nodes_per_replica: 1 +array_tasks: + count: 1 +submission: + job_name: product-descriptions + time_limit: 03:55:00 +output: + format: parquet + partitions: 1 + require_exact_record_count: false +``` + +`builder` accepts exactly one complete serialized Data Designer config through `source` or `inline`. Relative source paths resolve from the run file directory. `invocation.run_config` accepts public `RunConfig` fields, while `input_bindings.seed_path` and `managed_assets_path` accept absolute compute-visible paths. + +Every `model_alias` in `model_concurrency` and the builder must have one deployment. Deployment aliases must be unique. `model` selects the Hugging Face model ID or absolute model path loaded by the serving container. `served_model_name` is the name exposed to the client and defaults to `model`; set it explicitly when `model` is a container-visible path but the builder uses a stable model name. Output formats are `parquet`, `jsonl`, and `csv`. + +## Configure vLLM + +Each server supports readiness and launch controls: + +```yaml +server: + type: vllm + image: + name: vllm + startup_timeout: 20m + distributed_init_timeout: 10m + lead_boot_standoff: 60s + rank_launch_stagger: 5s + readiness_path: /health + queue_backpressure: + max_waiting_requests: 128 + retry_after_seconds: 1 + extra_args: + - --dtype + - bfloat16 +``` + +`distributed_init_timeout` must not exceed `startup_timeout`. Increase both when distributed initialization legitimately needs more time. + +`extra_args` is a list of individual arguments, not a shell command. Options owned by the orchestrator and unsafe shell forms are rejected. Configure model identity, topology, networking, and lifecycle through their typed fields instead of passing reserved options such as `--model`, `--served-model-name`, `--host`, `--port`, `--tensor-parallel-size`, `--pipeline-parallel-size`, or distributed-launch flags. In particular, `served_model_name` belongs on the deployment, alongside `model`, not under `server.extra_args`. + +## Scale runs + +### Arrays + +Split records into independent shards and limit how many allocations run at once: + +```yaml +invocation: + num_records: 100000 + dataset_name: large-run +array_tasks: + count: 20 + max_concurrent: 4 +output: + partitions: 20 +``` + +`count` cannot exceed `num_records`, and `max_concurrent` cannot exceed `count`. Status, retry, and collection retain shard identities and winners. + +Runs with more than one array task require Parquet output. They do not support global processors or profilers; image, plugin, custom, or unknown column types; local-callable validators; shuffled seed input; or authored seed-selection strategies. Seeded array runs must bind the dataset through `invocation.input_bindings.seed_path`. + +### Independent replicas + +Allocate multiple nodes and keep one vLLM replica per node: + +```yaml +resources: + nodes: 4 +topology: + tensor_parallel: 8 + nodes_per_replica: 1 +``` + +### Multi-node replicas + +Use more than one node for each replica: + +```yaml +resources: + nodes: 4 +topology: + tensor_parallel: 8 + nodes_per_replica: 2 +``` + +`tensor_parallel` must divide the GPUs available on each node. With the eight GPUs per node in the profile example, `tensor_parallel: 8` and `nodes_per_replica: 2` use 16 GPUs per replica. `nodes_per_replica` must divide `resources.nodes`. Multi-node expert parallelism is not supported in v1. + +### Multiple models and images + +Add one deployment per builder model alias. Each deployment may use a different registered serving image: + +```yaml +deployments: + - model_alias: writer + model: example/writer + server: + type: vllm + image: + name: vllm-writer + - model_alias: judge + model: example/judge + server: + type: vllm + image: + name: vllm-judge +invocation: + num_records: 1000 + dataset_name: judged-output + model_concurrency: + writer: 64 + judge: 32 +``` + +## Build a run in Python + +```python +from data_designer.config import DataDesignerConfigBuilder, LLMTextColumnConfig, ModelConfig +from data_designer.slurm.config import DataDesignerSlurmConfigBuilder + +builder = DataDesignerConfigBuilder( + model_configs=[ + ModelConfig(alias="generator", model="example/generator", provider="openai") + ] +) +builder.add_column( + LLMTextColumnConfig( + name="greeting", + prompt="Write one short greeting.", + model_alias="generator", + ) +) + +slurm = ( + DataDesignerSlurmConfigBuilder.from_config_builder(builder, name="generated-run") + .with_invocation( + num_records=100, + dataset_name="generated", + model_concurrency={"generator": 16}, + ) + .with_client(image={"name": "dd-client"}) + .with_deployment( + { + "model_alias": "generator", + "model": "example/generator", + "server": {"type": "vllm", "image": {"name": "vllm"}}, + } + ) +) +slurm.write_config("run.yaml") +``` + +The Slurm builder validates each orchestration step and writes strict YAML or JSON. Full Data Designer builder and plugin validation runs in the prepared client environment. The CLI rejects duplicate keys, YAML anchors, merge keys, unknown fields, and environment interpolation outside the serialized Data Designer builder payload. diff --git a/fern/versions/latest/pages/slurm/troubleshooting.mdx b/fern/versions/latest/pages/slurm/troubleshooting.mdx new file mode 100644 index 000000000..1ccc3e061 --- /dev/null +++ b/fern/versions/latest/pages/slurm/troubleshooting.mdx @@ -0,0 +1,84 @@ +# Slurm troubleshooting + +Start with the JSON error on stderr, then inspect the managed run directory and Slurm state. The CLI uses these exit codes: + +| Exit code | Error code | Meaning | +| --- | --- | --- | +| 1 | `internal` | Unexpected package failure | +| 2 | `invalid_request` | Invalid command input or configuration | +| 3 | `not_found` | Managed run, job, image, or benchmark not found | +| 4 | `conflict` | Existing immutable state conflicts with the request | +| 5 | `unavailable` | Slurm, storage, image, or runtime dependency unavailable | + +## The Slurm command group is missing + +Confirm the optional extra is installed in the active environment: + +```bash +python -m pip show data-designer +python -m pip show data-designer-slurm +data-designer slurm --help +``` + +Install `data-designer[slurm]` rather than only `data-designer`. The extension wheel must have the same version as the base wheel. If pip warns that the selected `data-designer` release does not provide the `slurm` extra, that package index or version does not contain the extension; select a release that publishes both packages instead of continuing with the base CLI. + +## Profile selection fails + +Pass both selectors explicitly while debugging: + +```bash +data-designer slurm profile validate \ + --profile-file /workspace/alice/config/slurm-profiles.yml \ + --cluster primary +``` + +Check that the profile file and workspace are accessible from the submit host, hostname patterns do not overlap, and `default_cluster` names an existing entry. Set `gpus_per_node` explicitly if an eligible partition contains heterogeneous nodes. + +## Slurm commands are unavailable in a batch job + +Set `scheduler.bin_path` in the selected profile to the absolute directory containing `srun`, `scontrol`, and the other Slurm executables. The generated batch script prepends this directory to its controlled `PATH`. + +## A small run requests every GPU on each node + +`gpus_per_node: auto` uses the GPU count reported by eligible nodes and requests that count in GRES mode. Set an explicit count such as `gpus_per_node: 1` when the cluster permits partial-node allocations, then keep deployment topology compatible with that count. + +## Image registration fails + +- Confirm the image-build partition accepts CPU jobs. +- Confirm Enroot is available on the allocated node. +- Use Enroot 4.0 or later for OCI import and 3.5 or later for existing SQSH inspection. +- Use a digest-qualified, credential-free OCI reference. +- Confirm the workspace path contains no whitespace, colon, comma, or backslash that Enroot mount syntax cannot represent. +- While a lifecycle job is active, inspect it under `/images/.tmp/jobs` and read its Slurm log. + +An interrupted or unknown submission can retain a temporary lifecycle job for investigation, but does not publish an unverified alias. Normal success and known failure paths clean temporary lifecycle state. + +## Dry-run cannot find an image + +Run `data-designer slurm image ls` against the same profile and cluster. Aliases are profile-workspace scoped. If the file changed after registration, add and inspect it again; planning rehashes registered SQSH content. + +## A run is absent from `squeue` + +Run `data-designer slurm status `. Completed jobs leave the active queue and may appear in `sacct` after a delay. Status reports accounting lag without inventing a terminal result. + +## Model readiness times out + +Inspect the attempt readiness record and Slurm stderr under the managed run directory. Confirm model files and mounts are visible, the serving image has the expected vLLM runtime, tensor parallelism matches available GPUs, and multi-node traffic is permitted. Increase `startup_timeout` only after identifying a normal slow startup rather than a failed launch. If increasing `distributed_init_timeout`, keep it less than or equal to `startup_timeout`. + +## A shard failed + +Use `status` to find the failed task and attempt. Batch stderr identifies the allocation runtime phase and exit status; synchronous step failures also report their durable stdout and stderr paths. Preview a sparse retry before submission: + +```bash +data-designer slurm retry --task-id --dry-run +``` + +Do not edit persisted manifests. Conflicts preserve scheduler and provenance evidence for operator review. + +## Output is incomplete + +Check that every shard has a `winner.json`, then follow its `candidate_manifest.path` to the canonical `output-manifest.json`. Do not count files by recursively scanning attempts. Retry missing shards, then use `merge` with the original run directory. When `require_exact_record_count` is true, a short result remains a failure rather than being silently accepted. + +## Scratch files are gone + +This is expected. Runtime extraction, client overlays, caches, and Enroot state live in allocation-local scratch and are cleaned on every terminal path. Use durable Slurm logs, attempt state, readiness records, and candidate results beneath `workspace_root` for diagnosis. diff --git a/packages/data-designer-slurm/README.md b/packages/data-designer-slurm/README.md index 04d29ccbd..1c000bd69 100644 --- a/packages/data-designer-slurm/README.md +++ b/packages/data-designer-slurm/README.md @@ -7,3 +7,5 @@ Install it through the Data Designer extra: ```bash pip install "data-designer[slurm]" ``` + +Follow the [Slurm Getting Started guide](https://docs.nvidia.com/nemo/datadesigner/slurm/getting-started) for profile setup, image registration, dry-run validation, submission, status, and result collection. Sanitized configuration examples are available in the [Data Designer repository](https://github.com/NVIDIA-NeMo/DataDesigner/tree/main/packages/data-designer-slurm/examples). diff --git a/packages/data-designer-slurm/examples/benchmark.yaml b/packages/data-designer-slurm/examples/benchmark.yaml new file mode 100644 index 000000000..120120654 --- /dev/null +++ b/packages/data-designer-slurm/examples/benchmark.yaml @@ -0,0 +1,27 @@ +schema_version: 1 +name: generator-scaling +base_run: run.yaml +model_aliases: + - generator +concurrency_values: + - 32 + - 64 +deployment_cases: + - name: single-node-groups + deployments: + generator: + nodes: 2 + nodes_per_replica: 1 + - name: two-node-group + deployments: + generator: + nodes: 2 + nodes_per_replica: 2 +record_policy: + type: adaptive + base_records: 1000 + max_records: 5000 + records_per_concurrency: 1.0 +analysis: + target_total_records: 1000000 + target_runtime: 4h diff --git a/packages/data-designer-slurm/examples/builder.yaml b/packages/data-designer-slurm/examples/builder.yaml new file mode 100644 index 000000000..9e2ca35d8 --- /dev/null +++ b/packages/data-designer-slurm/examples/builder.yaml @@ -0,0 +1,10 @@ +data_designer: + model_configs: + - alias: generator + model: example/generator + provider: openai + columns: + - name: greeting + column_type: llm-text + prompt: Write one short greeting. + model_alias: generator diff --git a/packages/data-designer-slurm/examples/profile-catalog.yaml b/packages/data-designer-slurm/examples/profile-catalog.yaml new file mode 100644 index 000000000..77ce34607 --- /dev/null +++ b/packages/data-designer-slurm/examples/profile-catalog.yaml @@ -0,0 +1,34 @@ +schema_version: 1 +default_cluster: primary +clusters: + primary: + schema_version: 1 + host_patterns: + - login*.example.test + scheduler: + account: research + partition: gpu + mem_per_gpu: 80G + bin_path: /opt/slurm/bin + gpus_per_node: 8 + workspace_root: /workspace/alice/data-designer-slurm + image_build: + partition: cpu + cpus_per_task: 2 + memory: 8G + time_limit: "04:00:00" + gpu_request_mode: gres + lab: + schema_version: 1 + host_patterns: + - lab-login*.example.test + scheduler: + partition: accelerated + gpus_per_node: 4 + workspace_root: /workspace/alice/data-designer-slurm-lab + image_build: + partition: general + cpus_per_task: 2 + memory: 8G + time_limit: "04:00:00" + gpu_request_mode: visible diff --git a/packages/data-designer-slurm/examples/run.yaml b/packages/data-designer-slurm/examples/run.yaml new file mode 100644 index 000000000..6879a4a96 --- /dev/null +++ b/packages/data-designer-slurm/examples/run.yaml @@ -0,0 +1,29 @@ +schema_version: 1 +name: greeting-run +builder: + source: builder.yaml +invocation: + num_records: 8 + dataset_name: greetings + model_concurrency: + generator: 8 +client: + image: + name: dd-client +deployments: + - model_alias: generator + served_model_name: example/generator + model: example/generator + server: + type: vllm + image: + name: vllm + startup_timeout: 15m + distributed_init_timeout: 10m + topology: + tensor_parallel: 1 +submission: + time_limit: 01:00:00 +output: + format: parquet + partitions: 1 diff --git a/packages/data-designer-slurm/src/data_designer/slurm/benchmark/execution.py b/packages/data-designer-slurm/src/data_designer/slurm/benchmark/execution.py index 6841a453c..871d88e2d 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/benchmark/execution.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/benchmark/execution.py @@ -122,7 +122,6 @@ def run( case.child_run_config, source_root=child_source_root, dry_run=False, - force=False, ) if execution.run_id != case.child_run_id or execution.state != "submitted": raise SlurmServiceError( diff --git a/packages/data-designer-slurm/src/data_designer/slurm/cli.py b/packages/data-designer-slurm/src/data_designer/slurm/cli.py index 18d8a4c2f..8a0ee3b27 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/cli.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/cli.py @@ -65,7 +65,6 @@ def execute_command( profile_file: Path | None = typer.Option(None, "--profile-file", dir_okay=False), cluster: str | None = typer.Option(None, "--cluster"), dry_run: bool = typer.Option(False, "--dry-run"), - force: bool = typer.Option(False, "--force"), ) -> None: """Prepare and submit one authored run.""" operation = SlurmServiceOperation.EXECUTE_RUN @@ -73,7 +72,7 @@ def execute_command( def execute() -> BaseModel: config = load_run_config(run_file) service = create_slurm_run_service(profile_file=profile_file, cluster=cluster) - return service.execute(config, source_root=run_file.resolve().parent, dry_run=dry_run, force=force) + return service.execute(config, source_root=run_file.resolve().parent, dry_run=dry_run) _emit_result(_invoke(operation, execute)) @@ -84,7 +83,7 @@ def status_command( profile_file: Path | None = typer.Option(None, "--profile-file", dir_okay=False), cluster: str | None = typer.Option(None, "--cluster"), ) -> None: - """Reconcile scheduler observations and show persisted M2 run status.""" + """Reconcile scheduler observations and show persisted run status.""" operation = SlurmServiceOperation.STATUS_RUN result = _invoke( operation, @@ -240,7 +239,7 @@ def image_add_command( profile_file: Path | None = typer.Option(None, "--profile-file", dir_okay=False), cluster: str | None = typer.Option(None, "--cluster"), ) -> None: - """Request image import or inspection; requires IMG lifecycle support.""" + """Import or inspect an image and register its alias.""" operation = SlurmServiceOperation.ADD_IMAGE def add() -> BaseModel: diff --git a/packages/data-designer-slurm/src/data_designer/slurm/config/run.py b/packages/data-designer-slurm/src/data_designer/slurm/config/run.py index 0d9380a92..f561a397d 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/config/run.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/config/run.py @@ -85,7 +85,7 @@ def validate_input(self) -> BuilderInput: validate_no_plaintext_secrets(self.inline, field_name="inline builder input") retired = {"dependencies", "sandbox_config", "server_configs"}.intersection(self.inline) if retired: - raise ValueError(f"builder input contains retired Big Iron fields: {', '.join(sorted(retired))}") + raise ValueError(f"builder input contains retired fields: {', '.join(sorted(retired))}") if "data_designer" in self.inline: unknown = set(self.inline).difference({"data_designer", "library_version"}) library_version = self.inline.get("library_version") diff --git a/packages/data-designer-slurm/src/data_designer/slurm/runtime/entrypoint.sh b/packages/data-designer-slurm/src/data_designer/slurm/runtime/entrypoint.sh index 327214f77..3a88d6515 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/runtime/entrypoint.sh +++ b/packages/data-designer-slurm/src/data_designer/slurm/runtime/entrypoint.sh @@ -11,6 +11,7 @@ source "${DD_RUNTIME_DIR}/cleanup.sh" DD_RUNTIME_PREPARED=0 DD_RUNTIME_FINALIZED=0 +DD_RUNTIME_PHASE=initialization DD_RUNTIME_RETRY_ARGUMENTS=() dd_slurm_run_allocation() { @@ -29,21 +30,28 @@ dd_slurm_run_allocation() { --effective-resume-mode "$5" ) fi + trap dd_runtime_exit EXIT + trap 'exit 130' INT TERM + DD_RUNTIME_PHASE=read_control_plan dd_read_control_plan "${DD_PLAN_PATH}" + DD_RUNTIME_PHASE=read_plan_secrets dd_read_plan_secret_names "${DD_PLAN_PATH}" + DD_RUNTIME_PHASE=translate_plan_path dd_read_container_path "${DD_PLAN_PATH}" "${DD_PLAN_PATH}" false DD_PLAN_CONTAINER_PATH=${DD_CONTAINER_PATH} + DD_RUNTIME_PHASE=translate_attempt_path dd_read_container_path "${DD_PLAN_PATH}" "${DD_ATTEMPT_PATH}" true DD_ATTEMPT_CONTAINER_DIR=${DD_CONTAINER_PATH} DD_RUNTIME_CONTAINER_ROOT=${DD_SCRATCH_CONTAINER_ROOT}/runtime + DD_RUNTIME_PHASE=translate_manifest_path dd_read_container_path "${DD_PLAN_PATH}" "${DD_RUNTIME_MANIFEST}" true DD_RUNTIME_MANIFEST_CONTAINER_PATH=${DD_CONTAINER_PATH} export DD_PLAN_PATH DD_ATTEMPT_PATH DD_RUNTIME_MANIFEST export DD_PLAN_CONTAINER_PATH DD_ATTEMPT_CONTAINER_DIR DD_RUNTIME_CONTAINER_ROOT - trap dd_runtime_exit EXIT - trap 'exit 130' INT TERM + DD_RUNTIME_PHASE=verify_host_context dd_verify_host_context + DD_RUNTIME_PHASE=start_runtime_timer dd_start_runtime_timer DD_RUNTIME_PREPARED=1 @@ -52,29 +60,40 @@ dd_slurm_run_allocation() { for host in "${DD_ALLOCATION_HOSTS[@]}"; do host_arguments+=(--node-host "${host}") done + DD_RUNTIME_PHASE=prepare dd_run_bound_control_phase prepare \ --runtime-root "${DD_RUNTIME_CONTAINER_ROOT}" \ --manifest "${DD_RUNTIME_MANIFEST_CONTAINER_PATH}" \ "${host_arguments[@]}" + DD_RUNTIME_PHASE=verify_runtime_manifest dd_verify_runtime_manifest \ "${DD_RUNTIME_MANIFEST}" \ "${DD_PLAN_SHA256}" \ "${DD_SHARD_ID}" \ "attempt-${DD_ATTEMPT_ORDINAL}" + DD_RUNTIME_PHASE=require_plan_secrets dd_require_plan_secrets + DD_RUNTIME_PHASE=client_preflight dd_read_step_ids "${DD_RUNTIME_MANIFEST}" client_preflight ((${#DD_STEP_IDS[@]} == 1)) dd_run_step "${DD_RUNTIME_MANIFEST}" "${DD_STEP_IDS[0]}" + DD_RUNTIME_PHASE=server_preflight dd_run_role_steps server_preflight + DD_RUNTIME_PHASE=start_servers dd_start_servers + DD_RUNTIME_PHASE=server_readiness dd_wait_for_role_readiness server + DD_RUNTIME_PHASE=start_endpoints dd_start_endpoints + DD_RUNTIME_PHASE=endpoint_readiness dd_wait_for_role_readiness endpoint dd_require_running + DD_RUNTIME_PHASE=ready dd_run_bound_control_phase ready + DD_RUNTIME_PHASE=client dd_read_step_ids "${DD_RUNTIME_MANIFEST}" client ((${#DD_STEP_IDS[@]} == 1)) dd_start_step "${DD_RUNTIME_MANIFEST}" "${DD_STEP_IDS[0]}" @@ -83,9 +102,12 @@ dd_slurm_run_allocation() { dd_wait_for_client "${client_pid}" dd_require_running + DD_RUNTIME_PHASE=cleanup dd_cleanup_steps + DD_RUNTIME_PHASE=succeed dd_run_bound_control_phase succeed DD_RUNTIME_FINALIZED=1 + DD_RUNTIME_PHASE=completed } dd_run_bound_control_phase() { @@ -236,6 +258,9 @@ dd_runtime_exit() { local status=$? trap - EXIT INT TERM set +e + if ((status != 0)); then + printf 'allocation runtime failed: phase=%s status=%d\n' "${DD_RUNTIME_PHASE}" "${status}" >&2 + fi dd_cleanup_steps if ((DD_RUNTIME_PREPARED == 1 && DD_RUNTIME_FINALIZED == 0)); then dd_run_bound_control_phase fail >/dev/null diff --git a/packages/data-designer-slurm/src/data_designer/slurm/runtime/step_runner.sh b/packages/data-designer-slurm/src/data_designer/slurm/runtime/step_runner.sh index 22c19afde..f1c0fb2b5 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/runtime/step_runner.sh +++ b/packages/data-designer-slurm/src/data_designer/slurm/runtime/step_runner.sh @@ -138,6 +138,10 @@ dd_run_step() { wait "${pid}" || status=$? unset "DD_MANAGED_PIDS[${index}]" DD_MANAGED_PIDS=("${DD_MANAGED_PIDS[@]+"${DD_MANAGED_PIDS[@]}"}") + if ((status != 0)); then + printf 'runtime step failed: step_id=%q status=%d stdout=%q stderr=%q\n' \ + "${step_id}" "${status}" "${DD_STEP_STDOUT}" "${DD_STEP_STDERR}" >&2 + fi return "${status}" } diff --git a/packages/data-designer-slurm/src/data_designer/slurm/services/results.py b/packages/data-designer-slurm/src/data_designer/slurm/services/results.py index 5d25a51f9..cecbbbf14 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/services/results.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/services/results.py @@ -83,7 +83,7 @@ def validate_shard(self) -> SlurmPersistedShardStatus: class SlurmPersistedRunStatus(ContractValue): - """M2 status assembled only from durable run records.""" + """Status assembled only from durable run records.""" run: RunManifest shards: tuple[SlurmPersistedShardStatus, ...] diff --git a/packages/data-designer-slurm/src/data_designer/slurm/services/run.py b/packages/data-designer-slurm/src/data_designer/slurm/services/run.py index f40ea2614..e241b3f4f 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/services/run.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/services/run.py @@ -63,12 +63,11 @@ def execute( *, source_root: Path, dry_run: bool, - force: bool, ) -> SlurmRunExecution: """Render or submit one run.""" def status(self, run_id: Identifier) -> SlurmPersistedRunStatus: - """Return the durable M2 status for one run.""" + """Return the durable status for one run.""" def cancel(self, run_id: Identifier) -> SlurmRunCancellation: """Request cancellation of active jobs.""" @@ -177,7 +176,6 @@ def execute( *, source_root: str | Path = ".", dry_run: bool = False, - force: bool = False, ) -> SlurmRunExecution: """Render or submit one run through package-owned production wiring.""" operation = SlurmServiceOperation.EXECUTE_RUN @@ -185,8 +183,8 @@ def execute( raise _make_invalid_request_error(operation, "config must be a DataDesignerSlurmConfig") if not isinstance(source_root, str | Path): raise _make_invalid_request_error(operation, "source_root must be a path") - if type(dry_run) is not bool or type(force) is not bool: - raise _make_invalid_request_error(operation, "dry_run and force must be booleans") + if type(dry_run) is not bool: + raise _make_invalid_request_error(operation, "dry_run must be a boolean") backend = self._require_backend(operation) def execute_run() -> SlurmRunExecution: @@ -194,7 +192,6 @@ def execute_run() -> SlurmRunExecution: config, source_root=Path(source_root).expanduser().resolve(), dry_run=dry_run, - force=force, ) if not isinstance(result, SlurmRunExecution): raise TypeError("run backend returned an invalid execution result") @@ -203,7 +200,7 @@ def execute_run() -> SlurmRunExecution: return _invoke_service_backend(operation, execute_run) def status(self, run_id: Identifier) -> SlurmPersistedRunStatus: - """Reconcile scheduler observations and return persisted M2 records.""" + """Reconcile scheduler observations and return persisted records.""" operation = SlurmServiceOperation.STATUS_RUN normalized_run_id = _validate_run_id(run_id, operation) backend = self._require_backend(operation) diff --git a/packages/data-designer-slurm/src/data_designer/slurm/services/wiring.py b/packages/data-designer-slurm/src/data_designer/slurm/services/wiring.py index 6055e6604..b4bfced83 100644 --- a/packages/data-designer-slurm/src/data_designer/slurm/services/wiring.py +++ b/packages/data-designer-slurm/src/data_designer/slurm/services/wiring.py @@ -298,7 +298,6 @@ def execute( *, source_root: Path, dry_run: bool, - force: bool, ) -> SlurmRunExecution: with self._preparer.prepare( config, @@ -321,7 +320,7 @@ def execute( publisher, config, prepared, - force=force, + force=False, ) try: receipt = self._launcher.submit_script( diff --git a/packages/data-designer-slurm/tests/benchmark/test_workflow.py b/packages/data-designer-slurm/tests/benchmark/test_workflow.py index 99e23af5f..735ba900f 100644 --- a/packages/data-designer-slurm/tests/benchmark/test_workflow.py +++ b/packages/data-designer-slurm/tests/benchmark/test_workflow.py @@ -64,9 +64,9 @@ def __init__(self, run_id, configs, submissions, calls, benchmark_root, failures self.benchmark_root = benchmark_root self.failures = failures - def execute(self, config, *, source_root, dry_run, force): + def execute(self, config, *, source_root, dry_run): assert (self.benchmark_root / "benchmark.json").is_file() - self.calls.append((self.run_id, config, source_root, dry_run, force)) + self.calls.append((self.run_id, config, source_root, dry_run)) self.configs[self.run_id] = config if self.run_id in self.failures: raise SlurmServiceError( @@ -89,9 +89,9 @@ def __init__(self, *args, second_check: Event) -> None: super().__init__(*args) self.second_check = second_check - def execute(self, config, *, source_root, dry_run, force): + def execute(self, config, *, source_root, dry_run): self.second_check.wait(timeout=1) - return super().execute(config, source_root=source_root, dry_run=dry_run, force=force) + return super().execute(config, source_root=source_root, dry_run=dry_run) class _ConcurrentSubmissionLoader: @@ -207,7 +207,6 @@ def test_run_persists_manifest_before_submission_and_is_idempotent( assert first == second assert len(calls) == len(compiled.cases) assert tuple(configs) == tuple(case.child_run_id for case in compiled.cases) - assert all(force is False for *_, force in calls) def test_concurrent_runs_submit_each_deterministic_child_once( diff --git a/packages/data-designer-slurm/tests/contracts/test_config_records.py b/packages/data-designer-slurm/tests/contracts/test_config_records.py index 58651662a..f74dc6a75 100644 --- a/packages/data-designer-slurm/tests/contracts/test_config_records.py +++ b/packages/data-designer-slurm/tests/contracts/test_config_records.py @@ -602,9 +602,11 @@ def test_run_rejects_retired_builder_fields(authored_run: DataDesignerSlurmConfi payload = authored_run.model_dump(mode="json") payload["builder"]["inline"]["server_configs"] = [] - with pytest.raises(ValidationError, match="retired"): + with pytest.raises(ValidationError, match="retired") as caught: DataDesignerSlurmConfig.model_validate(payload) + assert "builder input contains retired fields" in str(caught.value) + @pytest.mark.parametrize( "secret_key", diff --git a/packages/data-designer-slurm/tests/runtime/test_shell_runtime.py b/packages/data-designer-slurm/tests/runtime/test_shell_runtime.py index 649af45dd..b4cafd17d 100644 --- a/packages/data-designer-slurm/tests/runtime/test_shell_runtime.py +++ b/packages/data-designer-slurm/tests/runtime/test_shell_runtime.py @@ -336,6 +336,75 @@ def test_shell_helpers_handle_empty_and_sparse_arrays() -> None: assert completed.returncode == 0, completed.stderr +def test_allocation_runtime_reports_preparation_phase_and_preserves_status() -> None: + runtime_root = Path(__file__).parents[2] / "src/data_designer/slurm/runtime" + command = f""" +set -Eeuo pipefail +source {shlex.quote((runtime_root / "entrypoint.sh").as_posix())} +dd_read_control_plan() {{ return 4; }} +dd_cleanup_allocation_scratch() {{ :; }} +dd_cleanup_steps() {{ :; }} +dd_stop_runtime_timer() {{ :; }} +dd_slurm_run_allocation /workspace/resolved-plan.json /workspace/attempt +""" + + completed = subprocess.run(("bash", "-c", command), capture_output=True, text=True) + + assert completed.returncode == 4 + assert completed.stderr == "allocation runtime failed: phase=read_control_plan status=4\n" + + +def test_step_runner_reports_failed_step_log_paths(tmp_path: Path) -> None: + runtime_root = Path(__file__).parents[2] / "src/data_designer/slurm/runtime" + attempt_directory = tmp_path / "attempt" + log_directory = attempt_directory / "logs/execution-00000002" + log_directory.mkdir(parents=True) + image = tmp_path / "client.sqsh" + image.touch() + manifest = { + "steps": [ + _step( + attempt_directory, + "client-preflight", + "client_preflight", + image.as_posix(), + "true", + ) + ] + } + manifest_path = tmp_path / "runtime-manifest.json" + manifest_path.write_text(json.dumps(manifest)) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + _write_executable(fake_bin / "srun", "#!/usr/bin/env bash\nexit 4\n") + command = f""" +set -Eeuo pipefail +source {shlex.quote((runtime_root / "plan_reader.sh").as_posix())} +source {shlex.quote((runtime_root / "step_runner.sh").as_posix())} +DD_GPU_REQUEST_MODE=gres +DD_CONTAINER_MOUNTS= +DD_SCRATCH_ROOT=/tmp/data-designer-slurm-4101-0 +DD_SCRATCH_CONTAINER_ROOT=/run/data-designer-slurm +DD_ALL_SECRET_NAMES=() +DD_MANAGED_PIDS=() +status=0 +dd_run_step {shlex.quote(manifest_path.as_posix())} client-preflight || status=$? +exit "${{status}}" +""" + + completed = subprocess.run( + ("bash", "-c", command), + capture_output=True, + text=True, + env={**os.environ, "PATH": f"{fake_bin}:{os.environ['PATH']}"}, + ) + + assert completed.returncode == 4 + assert "runtime step failed: step_id=client-preflight status=4" in completed.stderr + assert f"stdout={log_directory / 'client-preflight.out'}" in completed.stderr + assert f"stderr={log_directory / 'client-preflight.err'}" in completed.stderr + + def test_step_runner_builds_one_coordinated_srun_across_selected_nodes() -> None: runtime_root = Path(__file__).parents[2] / "src/data_designer/slurm/runtime" command = f""" diff --git a/packages/data-designer-slurm/tests/services/test_services.py b/packages/data-designer-slurm/tests/services/test_services.py index df0b0e47e..b9b6855c9 100644 --- a/packages/data-designer-slurm/tests/services/test_services.py +++ b/packages/data-designer-slurm/tests/services/test_services.py @@ -194,7 +194,7 @@ def test_run_service_delegates_execute_actions( ) -> None: class RunBackend: def __init__(self) -> None: - self.calls: list[tuple[DataDesignerSlurmConfig, Path, bool, bool]] = [] + self.calls: list[tuple[DataDesignerSlurmConfig, Path, bool]] = [] def execute( self, @@ -202,9 +202,8 @@ def execute( *, source_root: Path, dry_run: bool, - force: bool, ) -> SlurmRunExecution: - self.calls.append((config, source_root, dry_run, force)) + self.calls.append((config, source_root, dry_run)) return SlurmRunExecution( run_id="run-0001", state="dry_run", @@ -222,10 +221,10 @@ def cancel(self, run_id): backend = RunBackend() service = SlurmRunService(FakeRunPlanningBackend(()), FakeBatchScriptRenderer(()), backend) - result = service.execute(authored_run_single, source_root=tmp_path, dry_run=True, force=True) + result = service.execute(authored_run_single, source_root=tmp_path, dry_run=True) assert result.run_id == "run-0001" - assert backend.calls == [(authored_run_single, tmp_path.resolve(), True, True)] + assert backend.calls == [(authored_run_single, tmp_path.resolve(), True)] def test_run_service_delegates_retry_with_stable_shard_order() -> None: diff --git a/packages/data-designer-slurm/tests/services/test_wiring.py b/packages/data-designer-slurm/tests/services/test_wiring.py index 02f072ebe..8af4bd7ae 100644 --- a/packages/data-designer-slurm/tests/services/test_wiring.py +++ b/packages/data-designer-slurm/tests/services/test_wiring.py @@ -385,11 +385,11 @@ def test_production_wiring_submits_after_publisher_initialization( package_version="0.9.2", ) - result = service.execute(authored_run_single, source_root=tmp_path, force=True) + result = service.execute(authored_run_single, source_root=tmp_path) assert result.state == "submitted" assert result.job_id == 42 - assert publisher.initializations == [("run-wired", True)] + assert publisher.initializations == [("run-wired", False)] assert publisher.submissions == [("run-wired", 42, submitted_at)] assert len(launcher.submissions) == 1 assert launcher.held_submissions == [True] @@ -398,28 +398,6 @@ def test_production_wiring_submits_after_publisher_initialization( assert (tmp_path / "managed-assets").is_dir() -def test_production_publisher_rejects_force_before_submission( - tmp_path: Path, - profile_catalog: SlurmProfileCatalog, - authored_run_single: DataDesignerSlurmConfig, - single_node_plan: ResolvedSlurmRunPlan, -) -> None: - _register_images(tmp_path, authored_run_single, single_node_plan) - launcher = _Launcher() - service = create_slurm_run_service( - profile=_profile(tmp_path, profile_catalog), - launcher=launcher, # type: ignore[arg-type] - run_id_factory=lambda: "run-wired", - package_version="0.9.2", - ) - - with pytest.raises(SlurmServiceError, match="different inputs") as caught: - service.execute(authored_run_single, source_root=tmp_path, force=True) - - assert caught.value.code is SlurmServiceErrorCode.CONFLICT - assert launcher.submissions == [] - - def test_production_wiring_exports_referenced_secrets_to_the_allocation( tmp_path: Path, profile_catalog: SlurmProfileCatalog, diff --git a/packages/data-designer-slurm/tests/test_cli.py b/packages/data-designer-slurm/tests/test_cli.py index dfa23f957..0cf0b83e1 100644 --- a/packages/data-designer-slurm/tests/test_cli.py +++ b/packages/data-designer-slurm/tests/test_cli.py @@ -26,7 +26,7 @@ class _RunService: def __init__(self) -> None: - self.calls: list[tuple[DataDesignerSlurmConfig, Path, bool, bool]] = [] + self.calls: list[tuple[DataDesignerSlurmConfig, Path, bool]] = [] self.retry_calls: list[tuple[str, tuple[str, ...] | None, str, bool]] = [] self.collection_calls: list[tuple[Path, Path, int | None]] = [] @@ -36,9 +36,8 @@ def execute( *, source_root: Path, dry_run: bool, - force: bool, ) -> SlurmRunExecution: - self.calls.append((config, source_root, dry_run, force)) + self.calls.append((config, source_root, dry_run)) return SlurmRunExecution( run_id="run-0001", state="dry_run", @@ -101,6 +100,20 @@ def analyze(self, benchmark_id, *, refresh_state, fail_if_incomplete): return self.report +@pytest.mark.parametrize( + ("arguments", "expected"), + [ + (["status", "--help"], "show persisted run status"), + (["image", "add", "--help"], "Import or inspect an image and register its alias"), + ], +) +def test_help_uses_public_terminology(arguments: list[str], expected: str) -> None: + result = CliRunner().invoke(cli_module.create_cli(), arguments) + + assert result.exit_code == 0 + assert expected in result.output + + def test_execute_emits_deterministic_json_and_forwards_actions( tmp_path: Path, authored_run_single: DataDesignerSlurmConfig, @@ -111,7 +124,7 @@ def test_execute_emits_deterministic_json_and_forwards_actions( service = _RunService() monkeypatch.setattr(cli_module, "create_slurm_run_service", lambda **_: service) - result = CliRunner().invoke(cli_module.create_cli(), ["execute", str(run_file), "--dry-run", "--force"]) + result = CliRunner().invoke(cli_module.create_cli(), ["execute", str(run_file), "--dry-run"]) assert result.exit_code == 0 assert json.loads(result.stdout) == { @@ -122,7 +135,7 @@ def test_execute_emits_deterministic_json_and_forwards_actions( "shard_count": 1, "state": "dry_run", } - assert service.calls == [(authored_run_single, tmp_path, True, True)] + assert service.calls == [(authored_run_single, tmp_path, True)] def test_benchmark_cli_forwards_run_and_analysis_actions( diff --git a/plans/874/slurm-documentation-coverage.md b/plans/874/slurm-documentation-coverage.md new file mode 100644 index 000000000..29356b4cd --- /dev/null +++ b/plans/874/slurm-documentation-coverage.md @@ -0,0 +1,36 @@ +# Slurm documentation coverage map + +This map records the behavioral migration from legacy Slurm guides to the public Fern Slurm guide. It tracks user-visible concepts rather than source wording or environment-specific examples. + +| Legacy area | Behavior retained in Fern | Fern destination | Disposition | +| --- | --- | --- | --- | +| Installation | Python environment, optional Slurm extra, CLI discovery | `overview.mdx`, `getting-started.mdx` | Retained and updated for the same-version optional wheel | +| Environment variables | Profile-file override, secret references, runtime environment boundaries | `profiles.mdx`, `dependencies-and-tools.mdx` | Relocated into the owning workflows | +| Cluster prerequisites | Submit commands, shared storage, Enroot, scheduler and GPU facts | `overview.mdx` | Expanded with workstation, login-host, and compute-node personas | +| Quick start | Install, profile, image import, builder, run config, dry-run, submit, status, cancel, output | `getting-started.mdx` | Replaced with a clean 13-step public flow | +| Profile configuration | Account, partitions, GPU discovery, GPU request mode, mounts | `profiles.mdx` | Expanded to a named multi-cluster catalog | +| Image management | OCI import, existing SQSH inspection, list, inspect, replace, remove | `images.mdx` | Expanded with digest and role verification rules | +| Client and serving images | Separate image roles and compatibility requirements | `images.mdx` | Retained and made explicit | +| Run configuration | Builder input, invocation, client, deployment, submission, output | `run-configuration.mdx` | Replaced with the shipped strict public schema | +| Server options | vLLM timeouts, readiness, backpressure, environment, safe arguments | `run-configuration.mdx` | Retained through supported public fields | +| Multi-node serving | Nodes, tensor parallelism, nodes per replica | `run-configuration.mdx` | Retained and expressed as deployment topology | +| Multiple models | Per-alias deployment, concurrency, and image selection | `run-configuration.mdx` | Retained and expanded to separate serving images | +| Job arrays | Shard count, concurrency throttle, persisted shard identity | `run-configuration.mdx` | Retained with public array constraints | +| Execute and observe | Dry-run, submit, JSON results, scheduler reconciliation | `operations.mdx` | Expanded with durable state behavior | +| Cancellation | Managed job ownership and idempotent cancellation | `operations.mdx` | Retained with run-ID ownership checks | +| Logs and results | Durable logs, attempt records, winners, default output | `operations.mdx` | Expanded with the persistent storage boundary | +| Runtime scratch | Runtime, dependency, cache, and Enroot placement and cleanup | `overview.mdx`, `operations.mdx`, `troubleshooting.mdx` | Intentional change to allocation-local scratch with stable container mount | +| Retry | Sparse task selection, resume policy, confirmation, dry-run | `retry-and-collection.mdx` | Expanded with immutable attempt history | +| Merge and collection | Winner-driven collection to an explicit destination | `retry-and-collection.mdx` | Retained as the public `merge` command | +| Dependencies | Pure-wheel overlay, immutable locks, client-image compatibility | `dependencies-and-tools.mdx` | Expanded with strict accepted sources | +| Plugins | Client image or dependency-overlay installation | `dependencies-and-tools.mdx` | Relocated and linked to public plugin contracts | +| Private packages and secrets | Built client images, immutable wheel or lock inputs, and external secret references | `dependencies-and-tools.mdx` | Replaced with the shipped public-index resolver and typed environment references | +| MCP providers | Remote and local stdio providers in client allocations | `dependencies-and-tools.mdx` | Added to match the shipped public surface | +| Code sandbox | Auxiliary isolated execution service | `dependencies-and-tools.mdx` | Intentionally unsupported in Slurm v1; alternatives documented | +| Benchmarks | Case expansion, child runs, status, point-in-time analysis | `benchmarks.mdx` | Retained and expanded with durable benchmark state | +| FAQ and recovery | Installation, profile, image, scheduler, readiness, retry, output, scratch | `troubleshooting.mdx` | Reorganized by observed failure | +| Command summary | Commands, common options, JSON output, exit codes | `cli-reference.mdx` | Replaced with a concise shipped-CLI reference | + +## Acceptance boundary + +The Fern guide and checked examples are validated in repository CI and against locally built wheels. Formal sealed-artifact acceptance, including the complete real-cluster scenario matrix and sanitized evidence bundle, remains issue #870 and must run after this documentation and release-integration slice merges and any required fixes land. diff --git a/scripts/audit_slurm_public_artifacts.py b/scripts/audit_slurm_public_artifacts.py index 236f8608b..41bdcbc85 100644 --- a/scripts/audit_slurm_public_artifacts.py +++ b/scripts/audit_slurm_public_artifacts.py @@ -4,7 +4,7 @@ """Scan public Slurm artifacts without rendering matched sensitive content. The default scope covers deployable source, test source and fixtures, package -metadata, and release scripts. A small path-scoped allowlist masks only exact +metadata, public documentation and examples, and release scripts. A small path-scoped allowlist masks only exact synthetic test sentinels. Generic example.test hosts, loopback addresses, and /workspace paths are the only implicit test-data allowances. """ @@ -32,9 +32,13 @@ "packages/data-designer-slurm/README.md", "packages/data-designer-slurm/pyproject.toml", "packages/data-designer-slurm/LICENSE", + "packages/data-designer-slurm/examples", "packages/data-designer-slurm/tests", + "fern/versions/latest/pages/slurm", + "fern/versions/latest.yml", "plans/850/data-designer-contract.md", "plans/870/slurm-early-security-review.md", + "plans/874/slurm-documentation-coverage.md", "scripts/publish.sh", "scripts/test_slurm_package_install.py", ) diff --git a/scripts/test_slurm_package_install.py b/scripts/test_slurm_package_install.py index 82a7cc4b5..7da8135ef 100644 --- a/scripts/test_slurm_package_install.py +++ b/scripts/test_slurm_package_install.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import os import shutil import statistics @@ -10,9 +11,14 @@ import sys import tempfile import time +from contextlib import contextmanager from email.message import Message from email.parser import BytesParser +from functools import partial +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from threading import Thread +from typing import Iterator from zipfile import ZipFile from packaging.requirements import Requirement @@ -25,15 +31,34 @@ "packages/data-designer", "packages/data-designer-slurm", ) +EXAMPLE_PATH = REPOSITORY_ROOT / "packages" / "data-designer-slurm" / "examples" +SLURM_DOCS_PATH = REPOSITORY_ROOT / "fern" / "versions" / "latest" / "pages" / "slurm" +FIRST_PARTY_PACKAGES = frozenset( + {"data-designer", "data-designer-config", "data-designer-engine", "data-designer-slurm"} +) CLI_HELP_SAMPLES = 9 MAX_BASE_CLI_HELP_SECONDS = 1.0 MAX_EXTENSION_CLI_HELP_OVERHEAD_SECONDS = 0.1 -def run(command: list[str], *, cwd: Path, check: bool = True) -> subprocess.CompletedProcess[str]: +class _QuietIndexHandler(SimpleHTTPRequestHandler): + def log_message(self, _format: str, *_args: object) -> None: + pass + + +def run( + command: list[str], + *, + cwd: Path, + check: bool = True, + isolate_pip: bool = False, +) -> subprocess.CompletedProcess[str]: environment = os.environ.copy() environment.pop("PYTHONPATH", None) environment.pop("VIRTUAL_ENV", None) + if isolate_pip: + environment = {name: value for name, value in environment.items() if not name.startswith("PIP_")} + environment["PIP_CONFIG_FILE"] = os.devnull result = subprocess.run(command, cwd=cwd, env=environment, capture_output=True, text=True, check=False) if check and result.returncode: raise RuntimeError(result.stdout + result.stderr) @@ -56,6 +81,19 @@ def audit_public_artifacts(*paths: Path) -> None: raise RuntimeError(result.stdout + result.stderr) +def verify_documented_examples() -> None: + documented_examples = { + "builder.yaml": "getting-started.mdx", + "run.yaml": "getting-started.mdx", + "benchmark.yaml": "benchmarks.mdx", + "profile-catalog.yaml": "profiles.mdx", + } + for name, document_name in documented_examples.items(): + example = (EXAMPLE_PATH / name).read_text().rstrip() + document = SLURM_DOCS_PATH.joinpath(document_name).read_text() + assert f"```yaml\n{example}\n```" in document + + def build_wheels(uv: str, wheel_directory: Path) -> dict[str, Path]: for package_path in PACKAGE_PATHS: run( @@ -95,6 +133,7 @@ def install(uv: str, python: Path, wheel_directory: Path, package: str, *, cwd: "--python", str(python), "--prerelease=allow", + "--no-index", "--find-links", str(wheel_directory), package, @@ -103,6 +142,79 @@ def install(uv: str, python: Path, wheel_directory: Path, package: str, *, cwd: ) +def download_third_party_wheels(wheel_directory: Path, metadata: dict[str, Message]) -> None: + requirements = { + str(parsed) + for distribution in metadata.values() + for value in distribution.get_all("Requires-Dist", []) + if canonicalize_name((parsed := Requirement(value)).name) not in FIRST_PARTY_PACKAGES + and (parsed.marker is None or parsed.marker.evaluate()) + } + run( + [ + sys.executable, + "-m", + "pip", + "download", + "--disable-pip-version-check", + "--only-binary=:all:", + "--pre", + "--dest", + str(wheel_directory), + *sorted(requirements), + ], + cwd=REPOSITORY_ROOT, + isolate_pip=True, + ) + + +def create_simple_index(root: Path, wheel_directory: Path) -> None: + wheels_by_package: dict[str, list[Path]] = {} + for wheel in wheel_directory.glob("*.whl"): + name = canonicalize_name(wheel_metadata(wheel)["Name"]) + wheels_by_package.setdefault(name, []).append(wheel) + for name, wheels in wheels_by_package.items(): + package_index = root / "simple" / name + package_index.mkdir(parents=True) + links = [] + for wheel in sorted(wheels): + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + links.append(f'{wheel.name}') + package_index.joinpath("index.html").write_text("".join(f"{link}\n" for link in links)) + + +@contextmanager +def serve_index(root: Path) -> Iterator[str]: + handler = partial(_QuietIndexHandler, directory=str(root)) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}/simple" + finally: + server.shutdown() + thread.join() + server.server_close() + + +def install_from_index(python: Path, index_url: str, package: str, *, cwd: Path) -> None: + run( + [ + str(python), + "-m", + "pip", + "install", + "--disable-pip-version-check", + "--pre", + "--index-url", + index_url, + package, + ], + cwd=cwd, + isolate_pip=True, + ) + + def verify_install(python: Path, version: str, *, slurm: bool, cwd: Path) -> None: statement = f""" import sys @@ -142,6 +254,7 @@ def verify_install(python: Path, version: str, *, slurm: bool, cwd: Path) -> Non assert "data_designer.slurm.cli" in sys.modules assert version("data-designer-slurm") == {version!r} from data_designer.slurm.benchmark import BenchmarkCompiler +from data_designer.slurm.config import load_benchmark_config, load_builder_payload, load_profile_catalog, load_run_config from data_designer.slurm.contracts import ArtifactReference as ContractArtifactReference from data_designer.slurm.contracts import RecordRange as ContractRecordRange from data_designer.slurm.contracts import ResumeWorkspace as ContractResumeWorkspace @@ -181,6 +294,14 @@ def verify_install(python: Path, version: str, *, slurm: bool, cwd: Path) -> Non assert StateArtifactReference is ContractArtifactReference assert StateRecordRange is ContractRecordRange assert StateResumeWorkspace is ContractResumeWorkspace +examples = Path("examples") +assert load_builder_payload(examples / "builder.yaml")["data_designer"]["columns"][0]["name"] == "greeting" +run_example = load_run_config(examples / "run.yaml") +benchmark_example = load_benchmark_config(examples / "benchmark.yaml") +assert run_example.name == "greeting-run" +assert benchmark_example.name == "generator-scaling" +assert len(BenchmarkCompiler.compile(benchmark_example, run_example).cases) == 4 +assert load_profile_catalog(examples / "profile-catalog.yaml").default_cluster == "primary" profile_help_result = CliRunner().invoke(app, ["slurm", "profile", "--help"]) assert profile_help_result.exit_code == 0, profile_help_result.output with TemporaryDirectory() as temporary_directory: @@ -233,27 +354,42 @@ def main() -> None: uv = shutil.which("uv") if uv is None: raise RuntimeError("uv is required") + if run([sys.executable, "-m", "pip", "--version"], cwd=REPOSITORY_ROOT, check=False).returncode: + raise RuntimeError("pip is required") + verify_documented_examples() with tempfile.TemporaryDirectory() as temporary_directory: root = Path(temporary_directory) + shutil.copytree(EXAMPLE_PATH, root / "examples") wheel_directory = root / "wheels" wheel_directory.mkdir() wheels = build_wheels(uv, wheel_directory) - base_wheel = wheels["data-designer"] + assert set(wheels) == FIRST_PARTY_PACKAGES + metadata = {name: wheel_metadata(path) for name, path in wheels.items()} + download_third_party_wheels(wheel_directory, metadata) + create_simple_index(root, wheel_directory) + versions = {item["Version"] for item in metadata.values()} + assert len(versions) == 1 leaf_wheel = wheels["data-designer-slurm"] audit_public_artifacts(leaf_wheel) - base_metadata = wheel_metadata(base_wheel) - leaf_metadata = wheel_metadata(leaf_wheel) + base_metadata = metadata["data-designer"] + leaf_metadata = metadata["data-designer-slurm"] version = base_metadata["Version"] assert leaf_metadata["Version"] == version + base_config_requirement = requirement(base_metadata, "data-designer-config") + base_engine_requirement = requirement(base_metadata, "data-designer-engine") + engine_config_requirement = requirement(metadata["data-designer-engine"], "data-designer-config") base_leaf_requirement = requirement(base_metadata, "data-designer-slurm") leaf_base_requirement = requirement(leaf_metadata, "data-designer") leaf_packaging_requirement = requirement(leaf_metadata, "packaging") leaf_pip_requirement = requirement(leaf_metadata, "pip") leaf_pydantic_requirement = requirement(leaf_metadata, "pydantic") leaf_pyyaml_requirement = requirement(leaf_metadata, "pyyaml") + assert str(base_config_requirement.specifier) == f"=={version}" + assert str(base_engine_requirement.specifier) == f"=={version}" + assert str(engine_config_requirement.specifier) == f"=={version}" assert str(base_leaf_requirement.specifier) == f"=={version}" assert str(leaf_base_requirement.specifier) == f"=={version}" assert leaf_packaging_requirement.specifier == Requirement("packaging>=25,<27").specifier @@ -272,7 +408,9 @@ def main() -> None: verify_install(base_python, version, slurm=False, cwd=root) extra_python = create_environment(uv, root / "extra", cwd=root) - install(uv, extra_python, wheel_directory, f"data-designer[slurm]=={version}", cwd=root) + install(uv, extra_python, wheel_directory, "pip", cwd=root) + with serve_index(root) as index_url: + install_from_index(extra_python, index_url, f"data-designer[slurm]=={version}", cwd=root) verify_install(extra_python, version, slurm=True, cwd=root) base_cli_help, extension_cli_help = cli_help_medians(base_python, extra_python, cwd=root) extension_overhead = extension_cli_help - base_cli_help