[RN50/MX] Skip first iterations for better metrics calculation

This commit is contained in:
Lukasz Pierscieniewski 2021-09-08 08:09:23 -07:00 committed by Krzysztof Kudrynski
parent 6d762dacdc
commit 88eb3cff2f
1 changed files with 3 additions and 1 deletions

View File

@ -234,6 +234,7 @@ def model_score(args, net, val_data, metric, kvstore):
tic = time.time()
metric = reduce_metrics(args, metric.get_global(), kvstore)
durations = durations[min(len(durations) // 10, 100):]
duration_stats = {
'ips': total_batch_size / np.mean(durations),
'latency_avg': np.mean(durations),
@ -365,9 +366,10 @@ def model_fit(args, net, train_data, eval_metric, optimizer,
durations.append(time.time() - tic)
tic = time.time()
durations = durations[min(len(durations) // 10, 100):]
dllogger_epoch_data = {
'train.loss': loss_metric.get_global()[1],
'train.ips': (i + 1) * total_batch_size / (time.time() - etic)
'train.ips': total_batch_size / np.mean(durations)
}
if args.mode == 'train_val':
logging.info('Validating epoch {}'.format(epoch))