From deb0349dacbf81bf4b9db65e3e9c52bbc690d070 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Tue, 18 Aug 2026 12:51:12 +0200 Subject: [PATCH 1/6] build: point lint tasks at pre-commit (#1369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build: point lint tasks at pre-commit `task lint` and `task format` ran `ruff` and `mypy .`, neither of which this repo enforces: there is no ruff config anywhere, ruff is absent from .pre-commit-config.yaml, and CI (.github/workflows/pre-commit.yml) runs pre-commit only. The result was that both tasks reported ~1300 findings and rewrote ~120 otherwise-clean files, and one ruff SIM117 autofix produced syntactically invalid code in tests/test_powermetrics.py. Both tasks now run pre-commit, matching CI exactly. On current master they pass clean. Also drops the unreferenced `ruff` dev dependency, fixes the `precommit` task (its command was the stray string "c"), and removes `docs-check-drift`, which pointed at scripts/check-docs-drift.py — a file that does not exist. Co-Authored-By: Claude Opus 5 (1M context) * build: drop the lint alias, it rewrites files Co-Authored-By: Claude Opus 5 (1M context) * build: keep the docs-check-drift task #1383 makes scripts/check_docs_drift.py exist and repoints this task at it, so deleting the task here would only fight that PR on the same line. Leave it untouched; this PR is about the ruff/mypy lint path. Co-Authored-By: Claude Opus 5 (1M context) * build: drop the duplicate precommit task precommit and format were the same command; keep format, the documented name. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: inimaz <49730431+inimaz@users.noreply.github.com> --- docs/how-to/agent-instructions.md | 8 +++----- pyproject.toml | 7 +++---- uv.lock | 27 --------------------------- 3 files changed, 6 insertions(+), 36 deletions(-) diff --git a/docs/how-to/agent-instructions.md b/docs/how-to/agent-instructions.md index 8b3ebfd0e..4486f158e 100644 --- a/docs/how-to/agent-instructions.md +++ b/docs/how-to/agent-instructions.md @@ -60,8 +60,7 @@ Here's what you need to know to navigate and contribute effectively. # Run specific test uv run pytest tests/test_emissions_tracker.py - # Lint and format - uv run task lint + # Lint and format (runs the pre-commit hooks, which rewrite files in place) uv run task format ``` @@ -112,7 +111,7 @@ Here's what you need to know to navigate and contribute effectively. 1. **Check existing tests** in `tests/` for similar functionality 2. **Add unit tests** first (test-driven development) 3. **Update documentation** if public interface changes -4. **Follow coding style**: Use `uv run task format` and `uv run task lint` +4. **Follow coding style**: Use `uv run task format` ### API Development 1. **Follow FastAPI patterns** - see routers in `carbonserver/carbonserver/api/routers/` @@ -134,8 +133,7 @@ uv run task -l # Main tasks: # - test-package: Core package testing -# - lint: Code linting and style checks -# - format: Code formatting +# - format: Lint and format, by running the pre-commit hooks # - test-api-unit: API unit tests # - test-api-integ: API integration tests # - dashboard: Run API locally diff --git a/pyproject.toml b/pyproject.toml index 26a360338..795be29b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,6 @@ dev = [ "taskipy", "bumpver", "pre-commit", - "ruff", "black", "mypy", "pytest", @@ -138,10 +137,10 @@ carbonserver-api-requirements = "uv pip compile carbonserver/pyproject.toml --ge build-doc = "uv run --only-group doc zensical build -f mkdocs.yml && uv run --only-group doc python scripts/check_docs_links.py site" precommit-install = "pre-commit install" precommit-update = "pre-commit autoupdate" -precommit = "c" mypy-check = "mypy -m codecarbon --ignore-missing-imports --no-strict-optional --disable-error-code attr-defined --disable-error-code assignment --disable-error-code misc" -lint = "black --check --diff . && ruff check . && mypy ." -format = "black . && ruff check --fix --exit-non-zero-on-fix ." +# No `lint` task: the pre-commit hooks fix what they can, so this rewrites +# files rather than only reporting. Use `mypy-check` for a read-only check. +format = "pre-commit run --all-files" test-package = "CODECARBON_ALLOW_MULTIPLE_RUNS=True pytest --ignore=tests/test_viz_data.py -vv -m 'not integ_test' tests/" test-coverage = "CODECARBON_ALLOW_MULTIPLE_RUNS=True pytest --cov --cov-report=xml --ignore=tests/test_viz_data.py -vv -m 'not integ_test' tests/" test-package-integ = "CODECARBON_ALLOW_MULTIPLE_RUNS=True python -m pytest -vv tests/" diff --git a/uv.lock b/uv.lock index f37050957..7830e8098 100644 --- a/uv.lock +++ b/uv.lock @@ -468,7 +468,6 @@ dev = [ { name = "requests" }, { name = "requests-mock" }, { name = "responses" }, - { name = "ruff" }, { name = "scikit-learn", version = "1.7.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scikit-learn", version = "1.9.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "taskipy" }, @@ -526,7 +525,6 @@ dev = [ { name = "requests" }, { name = "requests-mock" }, { name = "responses" }, - { name = "ruff" }, { name = "scikit-learn" }, { name = "taskipy" }, ] @@ -2853,31 +2851,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, ] -[[package]] -name = "ruff" -version = "0.16.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/70/25/7113f6d5498888c5fb7db34081cba7d5971c4cb1bfb26819966eee68f003/ruff-0.16.1.tar.gz", hash = "sha256:fedad7c801dabd3fb9741d76aca39246e6ddd9ca446a015875207bf19f1e6bc7", size = 4877500, upload-time = "2026-07-30T19:37:01.379Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/bd/694da69368e0973de65df2ddc73ab18d43c469d5963d9b150911de6bc513/ruff-0.16.1-py3-none-linux_armv6l.whl", hash = "sha256:58edb313b88f0c5460a26adf5f39a37a3be789494a15e3e411e35fa78b89f9a0", size = 10839126, upload-time = "2026-07-30T19:36:13.697Z" }, - { url = "https://files.pythonhosted.org/packages/3f/f0/b626e5d5bd0dd9576263658ef12885e2288afd1029a48e26ffed65ec1ac1/ruff-0.16.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fde5a99e2f97479af66edd6622c6d5a2a7592c77cf4153d9e4428f5eeb55b60c", size = 11070253, upload-time = "2026-07-30T19:36:17.14Z" }, - { url = "https://files.pythonhosted.org/packages/83/63/f40acfb6b35b88623e71684942b552c3edd96035f5d98f313815f7b277de/ruff-0.16.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e0d4c20532fca4f7fa609369161d968dd28f65d83dabbd61d8e9c7edbf7001f6", size = 10561425, upload-time = "2026-07-30T19:36:20.04Z" }, - { url = "https://files.pythonhosted.org/packages/aa/dd/14ec0e9c2b4d315547dd38765004b4863e354e1b52cb308272215d9f6f6d/ruff-0.16.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30affbcedf59ad5703d9c91f82266e02b47739f797e1a7b6e158e5526a6dae38", size = 10948879, upload-time = "2026-07-30T19:36:22.476Z" }, - { url = "https://files.pythonhosted.org/packages/33/e9/9d870cbae575030fdef595f04b4b97573c525b5497cce4f4498cf2f85446/ruff-0.16.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:24e9c631573cbca9d20f1283f8f479b2afa4a8503504822bd71a293889f16743", size = 10643691, upload-time = "2026-07-30T19:36:24.914Z" }, - { url = "https://files.pythonhosted.org/packages/c4/09/12743d544e2173f53ecd27217c65f90d2bc0f8424a66a60339e56bbc0457/ruff-0.16.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b41bdd48fb420987a9b5212e4957c26ad4abce401fa9ea9d4d85843727945f4f", size = 11435354, upload-time = "2026-07-30T19:36:28.447Z" }, - { url = "https://files.pythonhosted.org/packages/7f/89/a1652b2daee52083c9554a6333b678a8b01d0400f976827bb87857f9449a/ruff-0.16.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0d1e1393b7648079e13669de1c1f4fde06d4583e84d8fd5c1551e0a77a2aa75", size = 12259033, upload-time = "2026-07-30T19:36:31.326Z" }, - { url = "https://files.pythonhosted.org/packages/16/96/ecdcb8c54ee7b123b487f807eb014e6e019155a0b81dfb669acd52f28ce3/ruff-0.16.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:07bf434b1c95f4e093be4532068ef4fcf00924eb2ade8796075980902d6fd54a", size = 11667981, upload-time = "2026-07-30T19:36:34.394Z" }, - { url = "https://files.pythonhosted.org/packages/cd/90/c52e12e0d862e9572f2a33aa227409143520abe53111e9a6babbac7b4af8/ruff-0.16.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39897739f112253ee4fdd2e8aa9a4f9ded99fb2be367d5f31dfa4ded6025584c", size = 11468183, upload-time = "2026-07-30T19:36:37.339Z" }, - { url = "https://files.pythonhosted.org/packages/2c/6b/4ffb7ad1d83eb16cf8cbb3c8815d3f11c88460fd162d4b372a2059be1c2a/ruff-0.16.1-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:82ae3c0c0d74daf17b968a10b7b3bb3ef297ab7de0c1f749646b25e690ccb150", size = 11470071, upload-time = "2026-07-30T19:36:39.91Z" }, - { url = "https://files.pythonhosted.org/packages/9c/72/32ae7db4c0b5e32ab611787caa19d1546800676d79f7483b7100a3561bf4/ruff-0.16.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:4d5f2ed10f8242d83fc08d521301089364e3375375705356f20c0e31606ef3ef", size = 10919503, upload-time = "2026-07-30T19:36:42.65Z" }, - { url = "https://files.pythonhosted.org/packages/f7/ca/3d901ba6ad6fc38da39c3448fc6c59ac945679293a17c3ceb6d6c1cba13e/ruff-0.16.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a4665b309891f83f3e3c25447935f1213e9abbd4b5640af7a1f2def9f8d413c1", size = 10649861, upload-time = "2026-07-30T19:36:45.18Z" }, - { url = "https://files.pythonhosted.org/packages/92/79/894ef1ced26552d5f8c9cf6d85b0687840e1128c55aeab7b9c2d54a0d880/ruff-0.16.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:26e9ca5c9bc3971f20d3cf18a957f52ffd6a5f6564ff15c4912a144dcac22494", size = 11148137, upload-time = "2026-07-30T19:36:47.936Z" }, - { url = "https://files.pythonhosted.org/packages/2d/69/3609a09fa1cb46cc28b762363e440a354204e5dff01bd0c8d7437874d6b9/ruff-0.16.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:67e1e1e3fa4f0c82f0e36d4cd61e661f6e7a6196cb1aa92fe0828fa7b8f257cd", size = 11559211, upload-time = "2026-07-30T19:36:50.448Z" }, - { url = "https://files.pythonhosted.org/packages/fc/8a/fb22af2fd78a736e241fabf67e30ce1799a64244026377a49e133af90762/ruff-0.16.1-py3-none-win32.whl", hash = "sha256:d31765e131295b8445caf301e3e8a85b34d1b9b211b4109b7ba457888b051806", size = 10838258, upload-time = "2026-07-30T19:36:53.298Z" }, - { url = "https://files.pythonhosted.org/packages/d4/35/e57fd9fb5d423961df087a00b12d42c0a830288dc2f3b45ecca299158b4f/ruff-0.16.1-py3-none-win_amd64.whl", hash = "sha256:09b05e8b90c2cb06ad63464350e7a45e8e44a2dfe52072ebfba6666ca8d3f596", size = 11961111, upload-time = "2026-07-30T19:36:56.107Z" }, - { url = "https://files.pythonhosted.org/packages/cb/46/240ea004bf6dc4feb40e9832f2205a476a47dd5b8a3f8211a5fc5f95e20e/ruff-0.16.1-py3-none-win_arm64.whl", hash = "sha256:dbaadaac38c70239f056d306b7476f246b0bf000fa6b3876402acbf5b227eaf8", size = 11309414, upload-time = "2026-07-30T19:36:58.79Z" }, -] - [[package]] name = "scikit-learn" version = "1.7.2" From eb2cfef111ab21ac25ce5d2b665d94ad0a3ebb56 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Tue, 18 Aug 2026 16:57:21 +0200 Subject: [PATCH 2/6] fix: keep null coordinates when creating a run (#1330) `round(self.conf.get("longitude", 0), 1)` crashed with a TypeError whenever the key was present but None, which is the case for the offline tracker. The API output method then never created a run and silently dropped every emission. Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: inimaz <49730431+inimaz@users.noreply.github.com> --- codecarbon/core/api_client.py | 9 +++++++-- tests/test_api_call.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/codecarbon/core/api_client.py b/codecarbon/core/api_client.py index eaef94a53..bc2e0974e 100644 --- a/codecarbon/core/api_client.py +++ b/codecarbon/core/api_client.py @@ -22,6 +22,11 @@ from codecarbon.external.logger import logger +def _round_or_none(value: float | None) -> float | None: + """Round a coordinate, keeping None when it is unknown.""" + return None if value is None else round(value, 1) + + def get_datetime_with_timezone(): import arrow @@ -242,8 +247,8 @@ def _create_run(self, experiment_id: str): gpu_count=self.conf.get("gpu_count"), gpu_model=self.conf.get("gpu_model"), # Reduce precision for Privacy - longitude=round(self.conf.get("longitude", 0), 1), - latitude=round(self.conf.get("latitude", 0), 1), + longitude=_round_or_none(self.conf.get("longitude")), + latitude=_round_or_none(self.conf.get("latitude")), region=self.conf.get("region"), provider=self.conf.get("provider"), ram_total_size=self.conf.get("ram_total_size"), diff --git a/tests/test_api_call.py b/tests/test_api_call.py index d3b5bd96f..31e25c039 100644 --- a/tests/test_api_call.py +++ b/tests/test_api_call.py @@ -138,6 +138,39 @@ def test_call_api(self): assert payload["ram_utilization_percent"] == 56.5 assert payload["wue"] == 0.8 + def test_create_run_rounds_coordinates(self): + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-1"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="exp-1", + conf=conf, + create_run_automatically=False, + ) + + api._create_run("exp-1") + + payload = m.last_request.json() + self.assertEqual(payload["longitude"], -7.6) + self.assertEqual(payload["latitude"], 33.6) + + def test_create_run_keeps_unknown_coordinates_null(self): + offline_conf = dict(conf, longitude=None, latitude=None) + with requests_mock.Mocker() as m: + m.post("http://test.com/runs", json={"id": "run-1"}, status_code=201) + api = ApiClient( + endpoint_url="http://test.com", + experiment_id="exp-1", + conf=offline_conf, + create_run_automatically=False, + ) + + self.assertEqual(api._create_run("exp-1"), "run-1") + + payload = m.last_request.json() + self.assertIsNone(payload["longitude"]) + self.assertIsNone(payload["latitude"]) + def test_check_auth_raises_on_error(self): with requests_mock.Mocker() as m: m.get("http://test.com/auth/check", text="bad", status_code=401) From 88595c93b01e8bef57afce5939aca33d08fdd83e Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 18:08:01 +0200 Subject: [PATCH 3/6] fix: drop dtype coercion in CSV update path Updating a run row coerced each incoming value to the dtype pandas inferred for the existing column. Columns empty in every row are read back as float64, so numpy.float64("") / numpy.float64(None) raised. Rebuild the row via concat instead, which dedupes by run_id without touching dtypes. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/output_methods/file.py | 15 ++++----- tests/output_methods/test_file.py | 56 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/codecarbon/output_methods/file.py b/codecarbon/output_methods/file.py index 6bbf19a72..e613d225b 100644 --- a/codecarbon/output_methods/file.py +++ b/codecarbon/output_methods/file.py @@ -108,9 +108,7 @@ def out(self, total: EmissionsData, _): else: df = pd.read_csv(self.save_file_path) df_run = df.loc[df.run_id == total.run_id] - if len(df_run) < 1: - df = pd.concat([df, new_df]) - elif len(df_run) > 1: + if len(df_run) > 1: logger.warning( f"CSV contains more than 1 ({len(df_run)})" + f" rows with current run ID ({total.run_id})." @@ -118,12 +116,11 @@ def out(self, total: EmissionsData, _): ) df = pd.concat([df, new_df]) else: - update_values = {} - for col, val in dict(total.values).items(): - update_values[col] = df[col].dtype.type(val) - df.loc[df.run_id == total.run_id, update_values.keys()] = ( - update_values.values() - ) + # Drop the previous row for this run (if any) and re-append it. + # Assigning column by column would coerce values to the dtype + # pandas inferred for the existing column, which breaks for + # columns that are empty in every row (read back as float64). + df = pd.concat([df.loc[df.run_id != total.run_id], new_df]) df.to_csv(self.save_file_path, index=False) def task_out(self, data: List[TaskEmissionsData], experiment_name: str): diff --git a/tests/output_methods/test_file.py b/tests/output_methods/test_file.py index ec65cc3f0..d26d7c7a8 100644 --- a/tests/output_methods/test_file.py +++ b/tests/output_methods/test_file.py @@ -169,6 +169,62 @@ def test_file_output_out_update_file_exists_one_matchingrows(self): df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) self.assertEqual(df["cpu_power"].iloc[0], 2) + def test_file_output_out_update_with_always_empty_columns(self): + """Regression test: updating a run must not coerce incoming values to the + dtype pandas inferred for the existing column. + + An OfflineEmissionsTracker leaves longitude/latitude empty, and + gpu_count/gpu_model are empty on CPU-only machines. Such columns are read + back from the CSV as float64, so the previous implementation evaluated + numpy.float64("") / numpy.float64(None) and raised on the second write. + """ + empty_columns_data = EmissionsData( + timestamp="2023-01-01T00:00:00", + project_name="test_project", + run_id="test_run_id", + experiment_id="test_experiment_id", + duration=10, + emissions=0.5, + emissions_rate=0.05, + cpu_power=20, + gpu_power=0, + ram_power=5, + cpu_energy=200, + gpu_energy=0, + ram_energy=50, + energy_consumed=250, + water_consumed=0.1, + country_name="Testland", + country_iso_code="TS", + region="Test Region", + cloud_provider="", + cloud_region="", + os="TestOS", + python_version="3.8", + codecarbon_version="2.0", + cpu_count=4, + cpu_model="Test CPU", + gpu_count=None, + gpu_model=None, + longitude="", + latitude="", + ram_total_size=16, + tracking_mode="machine", + ) + + file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="update") + file_output.out(empty_columns_data, None) + + empty_columns_data.cpu_power = 2 + # This should not raise. + file_output.out(empty_columns_data, None) + + df = pd.read_csv(os.path.join(self.temp_dir, "test.csv")) + self.assertEqual(len(df), 1) + self.assertEqual(df["cpu_power"].iloc[0], 2) + self.assertIn("longitude", df.columns) + self.assertIn("gpu_model", df.columns) + # def test_file_output_out_consistent_column_ordering(self): # file_output = FileOutput("test.csv", self.temp_dir, on_csv_write="append") # file_output.out(self.emissions_data, None) From a6da2aca44e2f275c2a62a6c79af194256a1ecdc Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 17:55:30 +0200 Subject: [PATCH 4/6] feat: add scheduler job metadata to output Read the job identity SLURM already exports into every job step and store it on the emissions record, so an HPC job's rows are joinable against `sacct` instead of users smuggling the job id into `project_name`. Other schedulers map their own variables onto the same fields through `CODECARBON_SCHEDULER` / `CODECARBON_JOB_*`, which also override the auto-detected SLURM values. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/schedulers.py | 50 +++++++++++++ codecarbon/emissions_tracker.py | 2 + codecarbon/output_methods/emissions_data.py | 8 ++ docs/how-to/slurm.md | 51 +++++++++++++ docs/reference/output.md | 10 +++ tests/test_data/emissions_valid_headers.csv | 4 +- tests/test_schedulers.py | 83 +++++++++++++++++++++ 7 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 codecarbon/core/schedulers.py create mode 100644 tests/test_schedulers.py diff --git a/codecarbon/core/schedulers.py b/codecarbon/core/schedulers.py new file mode 100644 index 000000000..ebdd8e624 --- /dev/null +++ b/codecarbon/core/schedulers.py @@ -0,0 +1,50 @@ +""" +Detection of HPC batch scheduler job metadata. + +Schedulers export the identity of the running job into the environment of every +job step, so no scheduler library is needed: reading ``os.environ`` is enough. + +SLURM is detected automatically. Any other scheduler is supported through the +generic ``CODECARBON_JOB_*`` environment contract, which also takes precedence +over the auto-detected values, so a site can map its own scheduler in a couple +of lines of shell. +""" + +import os +from typing import Dict + +# Field name on EmissionsData -> SLURM environment variable holding it. +SLURM_ENV_VARS = { + "job_id": "SLURM_JOB_ID", + "job_name": "SLURM_JOB_NAME", + "job_user": "SLURM_JOB_USER", + "job_account": "SLURM_JOB_ACCOUNT", + "job_partition": "SLURM_JOB_PARTITION", + "node_name": "SLURMD_NODENAME", +} + +JOB_METADATA_FIELDS = ("scheduler",) + tuple(SLURM_ENV_VARS) + + +def detect_job_metadata() -> Dict[str, str]: + """ + Collect the scheduler job metadata of the current process. + + :return: a dict with one entry per field of ``JOB_METADATA_FIELDS``, with + empty strings for everything that could not be detected. Outside of a + batch job every value is empty. + """ + metadata = {field: "" for field in JOB_METADATA_FIELDS} + + if os.environ.get("SLURM_JOB_ID"): + metadata["scheduler"] = "slurm" + for field, env_var in SLURM_ENV_VARS.items(): + metadata[field] = os.environ.get(env_var, "") + + # Explicit configuration always wins, whatever the scheduler. + for field in JOB_METADATA_FIELDS: + override = os.environ.get(f"CODECARBON_{field.upper()}") + if override: + metadata[field] = override + + return metadata diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 96ed00c91..3b3a6dd90 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -21,6 +21,7 @@ from codecarbon._version import __version__ from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids +from codecarbon.core.schedulers import detect_job_metadata from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.hardware import CPU, GPU, AppleSiliconChip @@ -1098,6 +1099,7 @@ def _prepare_emissions_data(self) -> EmissionsData: tracking_mode=self._conf.get("tracking_mode"), pue=self._pue, wue=self._wue, + **detect_job_metadata(), ) logger.debug(total_emissions) return total_emissions diff --git a/codecarbon/output_methods/emissions_data.py b/codecarbon/output_methods/emissions_data.py index 17544aa51..2ca69fa47 100644 --- a/codecarbon/output_methods/emissions_data.py +++ b/codecarbon/output_methods/emissions_data.py @@ -47,6 +47,14 @@ class EmissionsData: on_cloud: str = "N" pue: float = 1 wue: float = 0 + # Batch scheduler job metadata, empty outside of an HPC job. + scheduler: str = "" + job_id: str = "" + job_name: str = "" + job_user: str = "" + job_account: str = "" + job_partition: str = "" + node_name: str = "" @property def values(self) -> OrderedDict: diff --git a/docs/how-to/slurm.md b/docs/how-to/slurm.md index 233097b8b..c3c364950 100644 --- a/docs/how-to/slurm.md +++ b/docs/how-to/slurm.md @@ -160,6 +160,57 @@ tail -f logs/.out sinfo ``` +## Job metadata in the output + +When CodeCarbon runs inside a SLURM job step it reads the job's identity from the +environment SLURM already provides and stores it on every emissions record. There is +nothing to enable and no code to change. + +| Output field | SLURM environment variable | +|--------------|----------------------------| +| `scheduler` | set to `slurm` when `SLURM_JOB_ID` is present | +| `job_id` | `SLURM_JOB_ID` | +| `job_name` | `SLURM_JOB_NAME` | +| `job_user` | `SLURM_JOB_USER` | +| `job_account` | `SLURM_JOB_ACCOUNT` | +| `job_partition` | `SLURM_JOB_PARTITION` | +| `node_name` | `SLURMD_NODENAME` | + +Outside of a job all of these are empty, so nothing changes for non-HPC users. + +This makes `emissions.csv` directly joinable against SLURM accounting: + +```bash +sacct -j 1234567 --format=JobID,JobName,Account,Partition,Elapsed,AllocTRES --parsable2 +``` + +!!! tip "You no longer need `CODECARBON_PROJECT_NAME=$SLURM_JOB_ID`" + Overloading the project name with the job ID used to be the only way to tell runs + apart. Keep `project_name` for your project and use `job_id` for the job. + +### Other schedulers + +Any field can be set, or overridden, with an environment variable named after it. This +is how PBS, LSF or OAR sites get the same columns without CodeCarbon needing to know +about their scheduler — map their variables onto ours in your job script: + +```bash +export CODECARBON_SCHEDULER=pbs +export CODECARBON_JOB_ID=$PBS_JOBID +export CODECARBON_JOB_NAME=$PBS_JOBNAME +export CODECARBON_NODE_NAME=$(hostname) +``` + +These variables take precedence over the auto-detected SLURM values, so they also work +for correcting a field on a site whose SLURM configuration is unusual. + +!!! warning "One tracker per node" + Power is a property of the node, not of a rank. If you launch CodeCarbon on every + rank of a multi-node job in `machine` tracking mode, each one measures the whole + node and your total is multiplied by the number of ranks. Start the tracker on one + rank per node (for example when `SLURM_LOCALID` is `0`), or use `process` tracking + mode. + ## Troubleshooting ### Error: AMD GPU detected but amdsmi is not properly configured diff --git a/docs/reference/output.md b/docs/reference/output.md index 720e0a883..3f40dbe71 100644 --- a/docs/reference/output.md +++ b/docs/reference/output.md @@ -64,6 +64,16 @@ The package has an in-built logger that logs data into a CSV file named `emissio | gpu_utilization_percent | Average GPU utilization during tracking period (%) | | ram_utilization_percent | Average RAM utilization during tracking period (%) | | ram_used_gb | Average RAM used during tracking period (GB) | +| scheduler | Batch scheduler that started the job, e.g. `slurm`. Empty outside of an HPC job | +| job_id | Scheduler job ID, e.g. the value of `SLURM_JOB_ID` | +| job_name | Scheduler job name | +| job_user | User the job runs as | +| job_account | Account/project the job is charged to | +| job_partition | Partition/queue the job runs in | +| node_name | Name of the compute node, as the scheduler knows it | + +The last seven fields are filled in automatically, see +[Using CodeCarbon on SLURM](../how-to/slurm.md#job-metadata-in-the-output). !!! note Developers can enhance the Output interface by implementing a custom class that extends `BaseOutput` at `codecarbon/output.py`. For example, to log into a database. diff --git a/tests/test_data/emissions_valid_headers.csv b/tests/test_data/emissions_valid_headers.csv index b7493c902..736e61ac3 100644 --- a/tests/test_data/emissions_valid_headers.csv +++ b/tests/test_data/emissions_valid_headers.csv @@ -1,2 +1,2 @@ -timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue -2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0 +timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue,scheduler,job_id,job_name,job_user,job_account,job_partition,node_name +2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0,,,,,,, diff --git a/tests/test_schedulers.py b/tests/test_schedulers.py new file mode 100644 index 000000000..7a53d5b15 --- /dev/null +++ b/tests/test_schedulers.py @@ -0,0 +1,83 @@ +import unittest +from unittest import mock + +from codecarbon.core.schedulers import JOB_METADATA_FIELDS, detect_job_metadata + +SLURM_ENV = { + "SLURM_JOB_ID": "1234567", + "SLURM_JOB_NAME": "train", + "SLURM_JOB_USER": "researcher", + "SLURM_JOB_ACCOUNT": "proj42", + "SLURM_JOB_PARTITION": "gpu", + "SLURMD_NODENAME": "nid001", +} + + +class TestSchedulers(unittest.TestCase): + @mock.patch.dict("os.environ", SLURM_ENV, clear=True) + def test_detect_slurm_env_parses_variables(self): + self.assertEqual( + detect_job_metadata(), + { + "scheduler": "slurm", + "job_id": "1234567", + "job_name": "train", + "job_user": "researcher", + "job_account": "proj42", + "job_partition": "gpu", + "node_name": "nid001", + }, + ) + + @mock.patch.dict("os.environ", {}, clear=True) + def test_no_scheduler_env_is_inert(self): + metadata = detect_job_metadata() + self.assertEqual(set(metadata), set(JOB_METADATA_FIELDS)) + self.assertEqual(set(metadata.values()), {""}) + + @mock.patch.dict("os.environ", {"SLURM_JOB_ID": "42"}, clear=True) + def test_partial_slurm_env_leaves_others_empty(self): + metadata = detect_job_metadata() + self.assertEqual(metadata["scheduler"], "slurm") + self.assertEqual(metadata["job_id"], "42") + self.assertEqual(metadata["job_name"], "") + + @mock.patch.dict( + "os.environ", + {"CODECARBON_SCHEDULER": "pbs", "CODECARBON_JOB_ID": "99.pbsserver"}, + clear=True, + ) + def test_generic_env_contract_without_slurm(self): + metadata = detect_job_metadata() + self.assertEqual(metadata["scheduler"], "pbs") + self.assertEqual(metadata["job_id"], "99.pbsserver") + + @mock.patch.dict( + "os.environ", dict(SLURM_ENV, CODECARBON_JOB_ACCOUNT="billed-to"), clear=True + ) + def test_generic_env_contract_overrides_slurm(self): + metadata = detect_job_metadata() + self.assertEqual(metadata["job_account"], "billed-to") + self.assertEqual(metadata["job_id"], "1234567") + + +class TestSchedulerMetadataOnEmissionsData(unittest.TestCase): + @mock.patch.dict("os.environ", SLURM_ENV, clear=True) + def test_job_fields_reach_the_emissions_data(self): + from codecarbon.emissions_tracker import OfflineEmissionsTracker + + tracker = OfflineEmissionsTracker( + country_iso_code="FRA", output_methods=[], allow_multiple_runs=True + ) + tracker.start() + try: + data = tracker._prepare_emissions_data() + finally: + tracker.stop() + + self.assertEqual(data.scheduler, "slurm") + self.assertEqual(data.job_id, "1234567") + self.assertEqual(data.job_account, "proj42") + self.assertEqual(data.node_name, "nid001") + # The new fields must be part of the CSV columns. + self.assertIn("job_partition", data.values) From 466892d4a4df4b8d15a25bf7911d39cad48c9986 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Wed, 12 Aug 2026 19:52:41 +0200 Subject: [PATCH 5/6] fix: warn on multi-rank double counting, note the CSV header change Rebased on fix/csv-update-dtype-coercion (#1370), which fixes the CSV dtype coercion properly, so the local workaround in file.py is dropped. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/schedulers.py | 24 ++++++++++++++++++++++++ codecarbon/emissions_tracker.py | 6 +++++- docs/reference/output.md | 7 +++++++ tests/test_schedulers.py | 26 +++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/codecarbon/core/schedulers.py b/codecarbon/core/schedulers.py index ebdd8e624..96297c437 100644 --- a/codecarbon/core/schedulers.py +++ b/codecarbon/core/schedulers.py @@ -11,8 +11,11 @@ """ import os +import re from typing import Dict +from codecarbon.external.logger import logger + # Field name on EmissionsData -> SLURM environment variable holding it. SLURM_ENV_VARS = { "job_id": "SLURM_JOB_ID", @@ -48,3 +51,24 @@ def detect_job_metadata() -> Dict[str, str]: metadata[field] = override return metadata + + +def warn_on_multi_rank_double_counting(tracking_mode: str) -> None: + """ + Warn when several ranks share a node and each one measures the whole node. + + In ``machine`` mode every rank reports the node's power, so the job's + reported footprint is silently multiplied by the number of ranks per node. + """ + if tracking_mode != "machine": + return + # SLURM writes this as "4", or as "4(x2)" for a heterogeneous allocation. + match = re.match(r"\d+", os.environ.get("SLURM_NTASKS_PER_NODE", "")) + if match and int(match.group()) > 1: + logger.warning( + f"SLURM_NTASKS_PER_NODE is {os.environ['SLURM_NTASKS_PER_NODE']} and " + "tracking_mode is 'machine': every rank measures the whole node, so " + "the job's total will be multiplied by the number of ranks per node. " + "Start the tracker on one rank per node (SLURM_LOCALID == 0), or use " + "tracking_mode='process'." + ) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 3b3a6dd90..733c5f716 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -21,7 +21,10 @@ from codecarbon._version import __version__ from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids -from codecarbon.core.schedulers import detect_job_metadata +from codecarbon.core.schedulers import ( + detect_job_metadata, + warn_on_multi_rank_double_counting, +) from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.hardware import CPU, GPU, AppleSiliconChip @@ -599,6 +602,7 @@ def __init__( ) assert self._tracking_mode in ["machine", "process"] + warn_on_multi_rank_double_counting(self._tracking_mode) set_logger_level(self._log_level) set_logger_format(self._logger_preamble) self._initialize_runtime_state() diff --git a/docs/reference/output.md b/docs/reference/output.md index 3f40dbe71..b3099d1e8 100644 --- a/docs/reference/output.md +++ b/docs/reference/output.md @@ -75,6 +75,13 @@ The package has an in-built logger that logs data into a CSV file named `emissio The last seven fields are filled in automatically, see [Using CodeCarbon on SLURM](../how-to/slurm.md#job-metadata-in-the-output). +!!! warning "Existing `emissions.csv` files are rotated once" + + These seven columns change the CSV header. On the first run after upgrading, + CodeCarbon backs up an existing `emissions.csv` next to it and starts a new + file with the new header. Nothing is lost, but a pipeline reading a fixed + path will see a file with only the new rows in it. + !!! note Developers can enhance the Output interface by implementing a custom class that extends `BaseOutput` at `codecarbon/output.py`. For example, to log into a database. diff --git a/tests/test_schedulers.py b/tests/test_schedulers.py index 7a53d5b15..f93e953e6 100644 --- a/tests/test_schedulers.py +++ b/tests/test_schedulers.py @@ -1,7 +1,11 @@ import unittest from unittest import mock -from codecarbon.core.schedulers import JOB_METADATA_FIELDS, detect_job_metadata +from codecarbon.core.schedulers import ( + JOB_METADATA_FIELDS, + detect_job_metadata, + warn_on_multi_rank_double_counting, +) SLURM_ENV = { "SLURM_JOB_ID": "1234567", @@ -61,6 +65,26 @@ def test_generic_env_contract_overrides_slurm(self): self.assertEqual(metadata["job_id"], "1234567") +class TestMultiRankWarning(unittest.TestCase): + def _warnings(self, tracking_mode="machine", **env): + with mock.patch.dict("os.environ", env, clear=True): + with mock.patch("codecarbon.core.schedulers.logger") as mocked_logger: + warn_on_multi_rank_double_counting(tracking_mode) + return mocked_logger.warning.call_count + + def test_several_ranks_per_node_in_machine_mode_warns(self): + self.assertEqual(1, self._warnings(SLURM_NTASKS_PER_NODE="4")) + # Heterogeneous allocations are written "4(x2)". + self.assertEqual(1, self._warnings(SLURM_NTASKS_PER_NODE="4(x2)")) + + def test_no_warning_without_double_counting(self): + self.assertEqual(0, self._warnings(SLURM_NTASKS_PER_NODE="1")) + self.assertEqual(0, self._warnings()) + self.assertEqual( + 0, self._warnings(tracking_mode="process", SLURM_NTASKS_PER_NODE="4") + ) + + class TestSchedulerMetadataOnEmissionsData(unittest.TestCase): @mock.patch.dict("os.environ", SLURM_ENV, clear=True) def test_job_fields_reach_the_emissions_data(self): From abb70b9fce63a2dc6e88d5fa40d747158f1a2e9a Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Sun, 16 Aug 2026 10:35:20 +0200 Subject: [PATCH 6/6] refactor: reduce job metadata to a single scheduler_job_id column Seven new columns rotate every user's emissions.csv on upgrade for a feature only HPC users can use. The job ID is the join key; sacct -j already knows the name, user, account, partition and node, so the other six columns buy nothing they cannot get for free. Also stop splatting **detect_job_metadata() into the EmissionsData constructor: any field-name drift became a runtime TypeError, and it re-read os.environ on every flush. The job identity cannot change during the process' life, so it is read once at init and passed as an explicit field. warn_on_multi_rank_double_counting() moved out to its own branch, it is unrelated scope. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/schedulers.py | 73 +++------------- codecarbon/emissions_tracker.py | 11 ++- codecarbon/output_methods/emissions_data.py | 10 +-- docs/how-to/slurm.md | 40 +++------ docs/reference/output.md | 14 +-- tests/test_data/emissions_valid_headers.csv | 4 +- tests/test_schedulers.py | 94 ++++----------------- 7 files changed, 56 insertions(+), 190 deletions(-) diff --git a/codecarbon/core/schedulers.py b/codecarbon/core/schedulers.py index 96297c437..8260c8714 100644 --- a/codecarbon/core/schedulers.py +++ b/codecarbon/core/schedulers.py @@ -1,74 +1,27 @@ """ -Detection of HPC batch scheduler job metadata. +Detection of the HPC batch scheduler job identity. Schedulers export the identity of the running job into the environment of every job step, so no scheduler library is needed: reading ``os.environ`` is enough. SLURM is detected automatically. Any other scheduler is supported through the -generic ``CODECARBON_JOB_*`` environment contract, which also takes precedence -over the auto-detected values, so a site can map its own scheduler in a couple -of lines of shell. +generic ``CODECARBON_SCHEDULER_JOB_ID`` environment variable, which also takes +precedence over the auto-detected value, so a site can map its own scheduler in +one line of shell. """ import os -import re -from typing import Dict -from codecarbon.external.logger import logger -# Field name on EmissionsData -> SLURM environment variable holding it. -SLURM_ENV_VARS = { - "job_id": "SLURM_JOB_ID", - "job_name": "SLURM_JOB_NAME", - "job_user": "SLURM_JOB_USER", - "job_account": "SLURM_JOB_ACCOUNT", - "job_partition": "SLURM_JOB_PARTITION", - "node_name": "SLURMD_NODENAME", -} - -JOB_METADATA_FIELDS = ("scheduler",) + tuple(SLURM_ENV_VARS) - - -def detect_job_metadata() -> Dict[str, str]: - """ - Collect the scheduler job metadata of the current process. - - :return: a dict with one entry per field of ``JOB_METADATA_FIELDS``, with - empty strings for everything that could not be detected. Outside of a - batch job every value is empty. - """ - metadata = {field: "" for field in JOB_METADATA_FIELDS} - - if os.environ.get("SLURM_JOB_ID"): - metadata["scheduler"] = "slurm" - for field, env_var in SLURM_ENV_VARS.items(): - metadata[field] = os.environ.get(env_var, "") - - # Explicit configuration always wins, whatever the scheduler. - for field in JOB_METADATA_FIELDS: - override = os.environ.get(f"CODECARBON_{field.upper()}") - if override: - metadata[field] = override - - return metadata - - -def warn_on_multi_rank_double_counting(tracking_mode: str) -> None: +def detect_scheduler_job_id() -> str: """ - Warn when several ranks share a node and each one measures the whole node. + Return the batch scheduler job ID of the current process, or "" outside of + a batch job. - In ``machine`` mode every rank reports the node's power, so the job's - reported footprint is silently multiplied by the number of ranks per node. + Only the job ID is collected: it is the join key, and everything else the + scheduler knows about the job (name, account, partition, node) is one + ``sacct -j `` away. """ - if tracking_mode != "machine": - return - # SLURM writes this as "4", or as "4(x2)" for a heterogeneous allocation. - match = re.match(r"\d+", os.environ.get("SLURM_NTASKS_PER_NODE", "")) - if match and int(match.group()) > 1: - logger.warning( - f"SLURM_NTASKS_PER_NODE is {os.environ['SLURM_NTASKS_PER_NODE']} and " - "tracking_mode is 'machine': every rank measures the whole node, so " - "the job's total will be multiplied by the number of ranks per node. " - "Start the tracker on one rank per node (SLURM_LOCALID == 0), or use " - "tracking_mode='process'." - ) + return os.environ.get("CODECARBON_SCHEDULER_JOB_ID") or os.environ.get( + "SLURM_JOB_ID", "" + ) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 733c5f716..0eeb37ea4 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -21,10 +21,7 @@ from codecarbon._version import __version__ from codecarbon.core.config import get_hierarchical_config, normalize_gpu_ids -from codecarbon.core.schedulers import ( - detect_job_metadata, - warn_on_multi_rank_double_counting, -) +from codecarbon.core.schedulers import detect_scheduler_job_id from codecarbon.core.units import Energy, Power, Time, Water from codecarbon.core.util import count_cpus, count_physical_cpus, suppress from codecarbon.external.hardware import CPU, GPU, AppleSiliconChip @@ -602,9 +599,11 @@ def __init__( ) assert self._tracking_mode in ["machine", "process"] - warn_on_multi_rank_double_counting(self._tracking_mode) set_logger_level(self._log_level) set_logger_format(self._logger_preamble) + # The job identity cannot change during the process' life, so it is read + # once here rather than on every flush. + self._scheduler_job_id = detect_scheduler_job_id() self._initialize_runtime_state() self._initialize_scheduler_state() self._initialize_emissions_context() @@ -1103,7 +1102,7 @@ def _prepare_emissions_data(self) -> EmissionsData: tracking_mode=self._conf.get("tracking_mode"), pue=self._pue, wue=self._wue, - **detect_job_metadata(), + scheduler_job_id=self._scheduler_job_id, ) logger.debug(total_emissions) return total_emissions diff --git a/codecarbon/output_methods/emissions_data.py b/codecarbon/output_methods/emissions_data.py index 2ca69fa47..33d3bd305 100644 --- a/codecarbon/output_methods/emissions_data.py +++ b/codecarbon/output_methods/emissions_data.py @@ -47,14 +47,8 @@ class EmissionsData: on_cloud: str = "N" pue: float = 1 wue: float = 0 - # Batch scheduler job metadata, empty outside of an HPC job. - scheduler: str = "" - job_id: str = "" - job_name: str = "" - job_user: str = "" - job_account: str = "" - job_partition: str = "" - node_name: str = "" + # Batch scheduler job ID, empty outside of an HPC job. + scheduler_job_id: str = "" @property def values(self) -> OrderedDict: diff --git a/docs/how-to/slurm.md b/docs/how-to/slurm.md index c3c364950..fab960a86 100644 --- a/docs/how-to/slurm.md +++ b/docs/how-to/slurm.md @@ -160,25 +160,15 @@ tail -f logs/.out sinfo ``` -## Job metadata in the output +## The job ID in the output -When CodeCarbon runs inside a SLURM job step it reads the job's identity from the -environment SLURM already provides and stores it on every emissions record. There is -nothing to enable and no code to change. +When CodeCarbon runs inside a SLURM job step it reads `SLURM_JOB_ID` from the +environment SLURM already provides and stores it on every emissions record, in the +`scheduler_job_id` column. There is nothing to enable and no code to change. Outside +of a job the column is empty, so nothing changes for non-HPC users. -| Output field | SLURM environment variable | -|--------------|----------------------------| -| `scheduler` | set to `slurm` when `SLURM_JOB_ID` is present | -| `job_id` | `SLURM_JOB_ID` | -| `job_name` | `SLURM_JOB_NAME` | -| `job_user` | `SLURM_JOB_USER` | -| `job_account` | `SLURM_JOB_ACCOUNT` | -| `job_partition` | `SLURM_JOB_PARTITION` | -| `node_name` | `SLURMD_NODENAME` | - -Outside of a job all of these are empty, so nothing changes for non-HPC users. - -This makes `emissions.csv` directly joinable against SLURM accounting: +This makes `emissions.csv` directly joinable against SLURM accounting, which knows +everything else about the job already: ```bash sacct -j 1234567 --format=JobID,JobName,Account,Partition,Elapsed,AllocTRES --parsable2 @@ -186,23 +176,19 @@ sacct -j 1234567 --format=JobID,JobName,Account,Partition,Elapsed,AllocTRES --pa !!! tip "You no longer need `CODECARBON_PROJECT_NAME=$SLURM_JOB_ID`" Overloading the project name with the job ID used to be the only way to tell runs - apart. Keep `project_name` for your project and use `job_id` for the job. + apart. Keep `project_name` for your project and use `scheduler_job_id` for the job. ### Other schedulers -Any field can be set, or overridden, with an environment variable named after it. This -is how PBS, LSF or OAR sites get the same columns without CodeCarbon needing to know -about their scheduler — map their variables onto ours in your job script: +Set `CODECARBON_SCHEDULER_JOB_ID` and the column is filled the same way. This is how +PBS, LSF or OAR sites get it without CodeCarbon needing to know about their scheduler: ```bash -export CODECARBON_SCHEDULER=pbs -export CODECARBON_JOB_ID=$PBS_JOBID -export CODECARBON_JOB_NAME=$PBS_JOBNAME -export CODECARBON_NODE_NAME=$(hostname) +export CODECARBON_SCHEDULER_JOB_ID=$PBS_JOBID ``` -These variables take precedence over the auto-detected SLURM values, so they also work -for correcting a field on a site whose SLURM configuration is unusual. +It takes precedence over the auto-detected SLURM value, so it also works for +correcting the field on a site whose SLURM configuration is unusual. !!! warning "One tracker per node" Power is a property of the node, not of a rank. If you launch CodeCarbon on every diff --git a/docs/reference/output.md b/docs/reference/output.md index b3099d1e8..9d9f1ab13 100644 --- a/docs/reference/output.md +++ b/docs/reference/output.md @@ -64,20 +64,14 @@ The package has an in-built logger that logs data into a CSV file named `emissio | gpu_utilization_percent | Average GPU utilization during tracking period (%) | | ram_utilization_percent | Average RAM utilization during tracking period (%) | | ram_used_gb | Average RAM used during tracking period (GB) | -| scheduler | Batch scheduler that started the job, e.g. `slurm`. Empty outside of an HPC job | -| job_id | Scheduler job ID, e.g. the value of `SLURM_JOB_ID` | -| job_name | Scheduler job name | -| job_user | User the job runs as | -| job_account | Account/project the job is charged to | -| job_partition | Partition/queue the job runs in | -| node_name | Name of the compute node, as the scheduler knows it | +| scheduler_job_id | Batch scheduler job ID, e.g. the value of `SLURM_JOB_ID`. Empty outside of an HPC job | -The last seven fields are filled in automatically, see -[Using CodeCarbon on SLURM](../how-to/slurm.md#job-metadata-in-the-output). +`scheduler_job_id` is filled in automatically, see +[Using CodeCarbon on SLURM](../how-to/slurm.md#the-job-id-in-the-output). !!! warning "Existing `emissions.csv` files are rotated once" - These seven columns change the CSV header. On the first run after upgrading, + This column changes the CSV header. On the first run after upgrading, CodeCarbon backs up an existing `emissions.csv` next to it and starts a new file with the new header. Nothing is lost, but a pipeline reading a fixed path will see a file with only the new rows in it. diff --git a/tests/test_data/emissions_valid_headers.csv b/tests/test_data/emissions_valid_headers.csv index 736e61ac3..060c21842 100644 --- a/tests/test_data/emissions_valid_headers.csv +++ b/tests/test_data/emissions_valid_headers.csv @@ -1,2 +1,2 @@ -timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue,scheduler,job_id,job_name,job_user,job_account,job_partition,node_name -2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0,,,,,,, +timestamp,project_name,run_id,experiment_id,duration,emissions,emissions_rate,cpu_power,gpu_power,ram_power,cpu_energy,gpu_energy,ram_energy,energy_consumed,water_consumed,country_name,country_iso_code,region,cloud_provider,cloud_region,os,python_version,codecarbon_version,cpu_count,cpu_model,gpu_count,gpu_model,longitude,latitude,ram_total_size,tracking_mode,cpu_utilization_percent,gpu_utilization_percent,ram_utilization_percent,ram_used_gb,on_cloud,pue,wue,scheduler_job_id +2021-09-23T15:04:51,codecarbon,0a578547-1d6b-4e2f-be0c-7ad10f2f7c97,test,161.20380687713623,0.0004490989249167,0.0027859076880178,0.269999999999999,0.0,12.884901888000002,0.0,0,0.00057442898176,0.00057442898176,0.1,Morocco,MAR,casablanca-settat,,,macOS-10.15.7-x86_64-i386-64bit,3.8.0,2.1.3,12,Intel(R) Core(TM) i7-8850H CPU @ 2.60GHz,,,-7.9084,33.5932,,machine,0.0,0.0,0.0,0.0,N,1.0,0.0, diff --git a/tests/test_schedulers.py b/tests/test_schedulers.py index f93e953e6..5748f0685 100644 --- a/tests/test_schedulers.py +++ b/tests/test_schedulers.py @@ -1,93 +1,36 @@ import unittest from unittest import mock -from codecarbon.core.schedulers import ( - JOB_METADATA_FIELDS, - detect_job_metadata, - warn_on_multi_rank_double_counting, -) - -SLURM_ENV = { - "SLURM_JOB_ID": "1234567", - "SLURM_JOB_NAME": "train", - "SLURM_JOB_USER": "researcher", - "SLURM_JOB_ACCOUNT": "proj42", - "SLURM_JOB_PARTITION": "gpu", - "SLURMD_NODENAME": "nid001", -} +from codecarbon.core.schedulers import detect_scheduler_job_id class TestSchedulers(unittest.TestCase): - @mock.patch.dict("os.environ", SLURM_ENV, clear=True) - def test_detect_slurm_env_parses_variables(self): - self.assertEqual( - detect_job_metadata(), - { - "scheduler": "slurm", - "job_id": "1234567", - "job_name": "train", - "job_user": "researcher", - "job_account": "proj42", - "job_partition": "gpu", - "node_name": "nid001", - }, - ) + @mock.patch.dict("os.environ", {"SLURM_JOB_ID": "1234567"}, clear=True) + def test_slurm_job_id_is_detected(self): + self.assertEqual("1234567", detect_scheduler_job_id()) @mock.patch.dict("os.environ", {}, clear=True) def test_no_scheduler_env_is_inert(self): - metadata = detect_job_metadata() - self.assertEqual(set(metadata), set(JOB_METADATA_FIELDS)) - self.assertEqual(set(metadata.values()), {""}) - - @mock.patch.dict("os.environ", {"SLURM_JOB_ID": "42"}, clear=True) - def test_partial_slurm_env_leaves_others_empty(self): - metadata = detect_job_metadata() - self.assertEqual(metadata["scheduler"], "slurm") - self.assertEqual(metadata["job_id"], "42") - self.assertEqual(metadata["job_name"], "") + self.assertEqual("", detect_scheduler_job_id()) @mock.patch.dict( - "os.environ", - {"CODECARBON_SCHEDULER": "pbs", "CODECARBON_JOB_ID": "99.pbsserver"}, - clear=True, + "os.environ", {"CODECARBON_SCHEDULER_JOB_ID": "99.pbsserver"}, clear=True ) def test_generic_env_contract_without_slurm(self): - metadata = detect_job_metadata() - self.assertEqual(metadata["scheduler"], "pbs") - self.assertEqual(metadata["job_id"], "99.pbsserver") + self.assertEqual("99.pbsserver", detect_scheduler_job_id()) @mock.patch.dict( - "os.environ", dict(SLURM_ENV, CODECARBON_JOB_ACCOUNT="billed-to"), clear=True + "os.environ", + {"SLURM_JOB_ID": "1234567", "CODECARBON_SCHEDULER_JOB_ID": "override"}, + clear=True, ) def test_generic_env_contract_overrides_slurm(self): - metadata = detect_job_metadata() - self.assertEqual(metadata["job_account"], "billed-to") - self.assertEqual(metadata["job_id"], "1234567") - - -class TestMultiRankWarning(unittest.TestCase): - def _warnings(self, tracking_mode="machine", **env): - with mock.patch.dict("os.environ", env, clear=True): - with mock.patch("codecarbon.core.schedulers.logger") as mocked_logger: - warn_on_multi_rank_double_counting(tracking_mode) - return mocked_logger.warning.call_count - - def test_several_ranks_per_node_in_machine_mode_warns(self): - self.assertEqual(1, self._warnings(SLURM_NTASKS_PER_NODE="4")) - # Heterogeneous allocations are written "4(x2)". - self.assertEqual(1, self._warnings(SLURM_NTASKS_PER_NODE="4(x2)")) - - def test_no_warning_without_double_counting(self): - self.assertEqual(0, self._warnings(SLURM_NTASKS_PER_NODE="1")) - self.assertEqual(0, self._warnings()) - self.assertEqual( - 0, self._warnings(tracking_mode="process", SLURM_NTASKS_PER_NODE="4") - ) + self.assertEqual("override", detect_scheduler_job_id()) -class TestSchedulerMetadataOnEmissionsData(unittest.TestCase): - @mock.patch.dict("os.environ", SLURM_ENV, clear=True) - def test_job_fields_reach_the_emissions_data(self): +class TestSchedulerJobIdOnEmissionsData(unittest.TestCase): + @mock.patch.dict("os.environ", {"SLURM_JOB_ID": "1234567"}, clear=True) + def test_job_id_reaches_the_emissions_data(self): from codecarbon.emissions_tracker import OfflineEmissionsTracker tracker = OfflineEmissionsTracker( @@ -99,9 +42,6 @@ def test_job_fields_reach_the_emissions_data(self): finally: tracker.stop() - self.assertEqual(data.scheduler, "slurm") - self.assertEqual(data.job_id, "1234567") - self.assertEqual(data.job_account, "proj42") - self.assertEqual(data.node_name, "nid001") - # The new fields must be part of the CSV columns. - self.assertIn("job_partition", data.values) + self.assertEqual("1234567", data.scheduler_job_id) + # The new field must be part of the CSV columns. + self.assertIn("scheduler_job_id", data.values)