Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions codecarbon/emissions_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,10 @@ def _resolve_output_methods(

def _initialize_runtime_state(self) -> None:
self._start_time: Optional[float] = None
# None while the tracker is running.
self._stopped_at: Optional[float] = None
self.final_emissions: Optional[float] = None
self.final_emissions_data: Optional[EmissionsData] = None
self._last_measured_time: float = time.perf_counter()
self._total_energy: Energy = Energy.from_energy(kWh=0)
self._total_emissions: float = 0.0
Expand Down Expand Up @@ -706,6 +710,14 @@ def start(self) -> None:
)
return
if self._start_time is not None:
if self._stopped_at is not None:
# `start()` is wrapped in @suppress(Exception), so raising here
# would be swallowed: log instead of pretending it worked.
logger.error(
"This tracker was already stopped and cannot be restarted. "
"Create a new tracker instead."
)
return
logger.warning("Already started tracking")
return

Expand Down Expand Up @@ -899,21 +911,24 @@ def stop(self) -> Optional[float]:
"Another instance of codecarbon is already running. Exiting."
)
return
if not self._allow_multiple_runs:
# Release the lock
self._lock.release()
if self._start_time is None:
logger.error("You first need to start the tracker.")
return None
if self._stopped_at is not None:
logger.warning("Tracker already stopped !")
return self.final_emissions
self._stopped_at = time.perf_counter()

if not self._allow_multiple_runs:
# Release the lock
self._lock.release()

if self._scheduler:
self._scheduler.stop()
self._scheduler = None
if self._scheduler_monitor_power:
self._scheduler_monitor_power.stop()
self._scheduler_monitor_power = None
else:
logger.warning("Tracker already stopped !")
for task_name in self._tasks:
if self._tasks[task_name].is_active:
self.stop_task(task_name=task_name)
Expand Down
73 changes: 73 additions & 0 deletions tests/test_emissions_tracker.py
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,62 @@ def test_offline_tracker_country_name(
self.assertEqual("United States", emissions_df["country_name"].values[0])
self.assertEqual("USA", emissions_df["country_iso_code"].values[0])

def test_offline_tracker_stop_is_idempotent(
self,
mock_cli_setup,
mock_log_values,
mocked_get_gpu_details,
mocked_env_cloud_details,
mocked_get_gpu_utilization_list,
mocked_is_gpu_details_available,
mocked_is_nvidia_system,
):
tracker = OfflineEmissionsTracker(
country_iso_code="USA",
output_dir=self.temp_path,
experiment_id="test",
)
tracker.start()
heavy_computation(run_time_secs=1)
first_emissions = tracker.stop()
second_emissions = tracker.stop()

self.assertEqual(first_emissions, second_emissions)
self.verify_output_file(self.emissions_file_path, 2)

def test_stop_releases_the_lock_only_once(
self,
mock_cli_setup,
mock_log_values,
mocked_get_gpu_details,
mocked_env_cloud_details,
mocked_get_gpu_utilization_list,
mocked_is_gpu_details_available,
mocked_is_nvidia_system,
):
with mock.patch("codecarbon.emissions_tracker.Lock") as mock_lock_class:
tracker = OfflineEmissionsTracker(
country_iso_code="USA",
output_dir=self.temp_path,
experiment_id="test",
allow_multiple_runs=False,
)
lock = mock_lock_class.return_value
lock.acquire.assert_called_once()

tracker.start()
heavy_computation(run_time_secs=1)
first_emissions = tracker.stop()
lock.release.assert_called_once()

# A second stop() is a no-op: it must not touch the lock again, which
# by then may belong to another tracker.
second_emissions = tracker.stop()
lock.release.assert_called_once()

self.assertEqual(first_emissions, second_emissions)
self.verify_output_file(self.emissions_file_path, 2)

def test_offline_tracker_invalid_headers(
self,
mock_cli_setup,
Expand Down Expand Up @@ -1108,3 +1164,20 @@ def test_cumulative_emissions_with_varying_intensity(

# Verification: If it wasn't cumulative, it would be 3.0 kWh * 300 g/kWh = 0.9 kg
self.assertLess(data3.emissions, 0.8)


class TestRestartAfterStop(unittest.TestCase):
def test_start_after_stop_is_refused(self):
tracker = OfflineEmissionsTracker(
country_iso_code="FRA",
save_to_file=False,
allow_multiple_runs=True,
measure_power_secs=10,
)
tracker.start()
tracker.stop()
with self.assertLogs("codecarbon", level="ERROR") as logs:
tracker.start()
self.assertIn("cannot be restarted", "".join(logs.output))
# Refused, not half-restarted: nothing was rebuilt.
self.assertIsNone(tracker._scheduler)
Loading