From 4425abce59ebd87b6b1426419212a3de714cffcd Mon Sep 17 00:00:00 2001 From: Finlay Morrison <57669260+finlaymorrison@users.noreply.github.com> Date: Fri, 1 Dec 2023 15:45:31 +0000 Subject: [PATCH] Update Tuner() attempt metrics save on crash (#6711) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- ultralytics/engine/tuner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ultralytics/engine/tuner.py b/ultralytics/engine/tuner.py index 1e5090ab..f729b0f7 100644 --- a/ultralytics/engine/tuner.py +++ b/ultralytics/engine/tuner.py @@ -180,8 +180,9 @@ class Tuner: try: # Train YOLO model with mutated hyperparameters (run in subprocess to avoid dataloader hang) cmd = ['yolo', 'train', *(f'{k}={v}' for k, v in train_args.items())] - assert subprocess.run(cmd, check=True).returncode == 0, 'training failed' + return_code = subprocess.run(cmd, check=True).returncode metrics = torch.load(ckpt_file)['train_metrics'] + assert return_code == 0, 'training failed' except Exception as e: LOGGER.warning(f'WARNING ❌️ training failure for hyperparameter tuning iteration {i + 1}\n{e}')