Skip to content

Commit

Permalink
Merge pull request #219 from the-database/dev
Browse files Browse the repository at this point in the history
fix console log rendering
  • Loading branch information
the-database authored Jan 7, 2025
2 parents 5fc0183 + c06e6aa commit 8a28d69
Showing 1 changed file with 5 additions and 4 deletions.
9 changes: 5 additions & 4 deletions traiNNer/utils/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,8 +112,8 @@ def __call__(self, log_vars: dict[str, Any]) -> None:
lrs = log_vars.pop("lrs")

# Construct the base message with epoch, iteration, and learning rates
message = f"[epoch:{epoch:4,d}, iter:{current_iter:8,d}, lr:("
message += ", ".join([f"{v:.3e}" for v in lrs]) + ")] "
message = f"[ epoch:{epoch:4,d}, iter:{current_iter:8,d}, lr:("
message += ", ".join([f"{v:.3e}" for v in lrs]) + ") ] "

# performance, eta
if "time" in log_vars.keys():
Expand All @@ -125,11 +125,11 @@ def __call__(self, log_vars: dict[str, Any]) -> None:
eta_sec = time_sec_avg * (self.max_iters - current_iter - 1)
eta_str = str(datetime.timedelta(seconds=int(eta_sec)))

message += f"[performance: {iter_time:.3f} it/s] [eta: {eta_str}] "
message += f"[ performance: {iter_time:.3f} it/s ] [ eta: {eta_str} ] "

# peak VRAM
message += (
f"[peak VRAM: {torch.cuda.max_memory_allocated() / (1024**3):.2f} GB] "
f"[ peak VRAM: {torch.cuda.max_memory_allocated() / (1024**3):.2f} GB ] "
)

# Log any additional variables (typically losses)
Expand All @@ -141,6 +141,7 @@ def __call__(self, log_vars: dict[str, Any]) -> None:
self.tb_logger.add_scalar(label, value, current_iter)

# Log the final constructed message

self.logger.info(message)


Expand Down

0 comments on commit 8a28d69

Please sign in to comment.