mirror of
https://github.com/THU-MIG/yolov10.git
synced 2025-05-23 13:34:23 +08:00
b54055a2c7
Co-authored-by: Hassaan Farooq <103611273+hassaanfarooq01@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Glenn Jocher <glenn.jocher@ultralytics.com>
99 lines
3.3 KiB
Python
99 lines
3.3 KiB
Python
# Ultralytics YOLO 🚀, AGPL-3.0 license
|
|
|
|
import json
|
|
from time import time
|
|
|
|
from hub_sdk.config import HUB_WEB_ROOT
|
|
|
|
from ultralytics.hub.utils import PREFIX, events
|
|
from ultralytics.utils import LOGGER, SETTINGS
|
|
|
|
|
|
def on_pretrain_routine_end(trainer):
|
|
"""Logs info before starting timer for upload rate limit."""
|
|
session = getattr(trainer, 'hub_session', None)
|
|
if session:
|
|
# Start timer for upload rate limit
|
|
session.timers = {
|
|
'metrics': time(),
|
|
'ckpt': time(), } # start timer on session.rate_limit
|
|
|
|
|
|
def on_fit_epoch_end(trainer):
|
|
"""Uploads training progress metrics at the end of each epoch."""
|
|
session = getattr(trainer, 'hub_session', None)
|
|
if session:
|
|
# Upload metrics after val end
|
|
all_plots = {
|
|
**trainer.label_loss_items(trainer.tloss, prefix='train'),
|
|
**trainer.metrics, }
|
|
if trainer.epoch == 0:
|
|
from ultralytics.utils.torch_utils import model_info_for_loggers
|
|
all_plots = {**all_plots, **model_info_for_loggers(trainer)}
|
|
|
|
session.metrics_queue[trainer.epoch] = json.dumps(all_plots)
|
|
if time() - session.timers['metrics'] > session.rate_limits['metrics']:
|
|
session.upload_metrics()
|
|
session.timers['metrics'] = time() # reset timer
|
|
session.metrics_queue = {} # reset queue
|
|
|
|
|
|
def on_model_save(trainer):
|
|
"""Saves checkpoints to Ultralytics HUB with rate limiting."""
|
|
session = getattr(trainer, 'hub_session', None)
|
|
if session:
|
|
# Upload checkpoints with rate limiting
|
|
is_best = trainer.best_fitness == trainer.fitness
|
|
if time() - session.timers['ckpt'] > session.rate_limits['ckpt']:
|
|
LOGGER.info(f'{PREFIX}Uploading checkpoint {HUB_WEB_ROOT}/models/{session.model_file}')
|
|
session.upload_model(trainer.epoch, trainer.last, is_best)
|
|
session.timers['ckpt'] = time() # reset timer
|
|
|
|
|
|
def on_train_end(trainer):
|
|
"""Upload final model and metrics to Ultralytics HUB at the end of training."""
|
|
session = getattr(trainer, 'hub_session', None)
|
|
if session:
|
|
# Upload final model and metrics with exponential standoff
|
|
LOGGER.info(f'{PREFIX}Syncing final model...')
|
|
session.upload_model(
|
|
trainer.epoch,
|
|
trainer.best,
|
|
map=trainer.metrics.get('metrics/mAP50-95(B)', 0),
|
|
final=True,
|
|
)
|
|
session.alive = False # stop heartbeats
|
|
LOGGER.info(f'{PREFIX}Done ✅\n'
|
|
f'{PREFIX}View model at {session.model_url} 🚀')
|
|
|
|
|
|
def on_train_start(trainer):
|
|
"""Run events on train start."""
|
|
events(trainer.args)
|
|
|
|
|
|
def on_val_start(validator):
|
|
"""Runs events on validation start."""
|
|
events(validator.args)
|
|
|
|
|
|
def on_predict_start(predictor):
|
|
"""Run events on predict start."""
|
|
events(predictor.args)
|
|
|
|
|
|
def on_export_start(exporter):
|
|
"""Run events on export start."""
|
|
events(exporter.args)
|
|
|
|
|
|
callbacks = ({
|
|
'on_pretrain_routine_end': on_pretrain_routine_end,
|
|
'on_fit_epoch_end': on_fit_epoch_end,
|
|
'on_model_save': on_model_save,
|
|
'on_train_end': on_train_end,
|
|
'on_train_start': on_train_start,
|
|
'on_val_start': on_val_start,
|
|
'on_predict_start': on_predict_start,
|
|
'on_export_start': on_export_start, } if SETTINGS['hub'] is True else {}) # verify enabled
|