diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 4bd0ea99d..e1254a4a7 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -268,6 +268,8 @@ def _resolve_output_methods( def _initialize_runtime_state(self) -> None: self._start_time: Optional[float] = None + # Set by stop(): a stopped tracker cannot be restarted. + self._stopped: bool = False self._last_measured_time: float = time.perf_counter() self._total_energy: Energy = Energy.from_energy(kWh=0) self._total_emissions: float = 0.0 @@ -708,6 +710,14 @@ def start(self) -> None: "Another instance of codecarbon is already running. Exiting." ) return + if self._stopped: + # `start()` is wrapped in @suppress(Exception), so raising here + # would be swallowed: log instead of pretending it worked. + logger.error( + "This tracker was already stopped and cannot be restarted: " + "nothing will be measured. Create a new tracker instead." + ) + return if self._start_time is not None: logger.warning("Already started tracking") return @@ -745,6 +755,12 @@ def start_task(self, task_name=None) -> None: "Another instance of codecarbon is already running. Exiting." ) return + if self._stopped: + logger.error( + "This tracker was already stopped and cannot be restarted: " + "nothing will be measured. Create a new tracker instead." + ) + return try: self._ensure_emissions_engine() except Exception: @@ -965,6 +981,7 @@ def stop(self) -> Optional[float]: handler.exit() self._start_time = None + self._stopped = True return emissions_data.emissions diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index a2d6b5966..0a38fc2ac 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -633,6 +633,39 @@ def test_offline_tracker_country_name( self.assertEqual("United States", emissions_df["country_name"].values[0]) self.assertEqual("USA", emissions_df["country_iso_code"].values[0]) + def test_stop_releases_the_lock_only_once( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_get_gpu_utilization_list, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + with mock.patch("codecarbon.emissions_tracker.Lock") as mock_lock_class: + tracker = OfflineEmissionsTracker( + country_iso_code="USA", + output_dir=self.temp_path, + experiment_id="test", + allow_multiple_runs=False, + ) + lock = mock_lock_class.return_value + lock.acquire.assert_called_once() + + tracker.start() + heavy_computation(run_time_secs=1) + first_emissions = tracker.stop() + lock.release.assert_called_once() + + # A second stop() is a no-op: it must not touch the lock again, which + # by then may belong to another tracker. + second_emissions = tracker.stop() + lock.release.assert_called_once() + + self.assertEqual(first_emissions, second_emissions) + self.verify_output_file(self.emissions_file_path, 2) + def test_offline_tracker_invalid_headers( self, mock_cli_setup, @@ -1153,3 +1186,36 @@ def test_cumulative_emissions_with_varying_intensity( # Verification: If it wasn't cumulative, it would be 3.0 kWh * 300 g/kWh = 0.9 kg self.assertLess(data3.emissions, 0.8) + + +class TestRestartAfterStop(unittest.TestCase): + def test_start_after_stop_is_refused(self): + tracker = OfflineEmissionsTracker( + country_iso_code="FRA", + save_to_file=False, + allow_multiple_runs=True, + measure_power_secs=10, + ) + tracker.start() + tracker.stop() + with self.assertLogs("codecarbon", level="ERROR") as logs: + tracker.start() + self.assertIn("cannot be restarted", "".join(logs.output)) + # Refused, not half-restarted: nothing was rebuilt. + self.assertIsNone(tracker._scheduler) + + def test_start_task_after_stop_is_refused(self): + tracker = OfflineEmissionsTracker( + country_iso_code="FRA", + save_to_file=False, + allow_multiple_runs=True, + measure_power_secs=10, + ) + tracker.start() + tracker.stop() + with self.assertLogs("codecarbon", level="ERROR") as logs: + # _scheduler is None after stop(); start_task must hit the + # _stopped guard before touching it, not raise AttributeError. + tracker.start_task() + self.assertIn("cannot be restarted", "".join(logs.output)) + self.assertIsNone(tracker._active_task)