a55c6778-e76a-44bc-a0e1-68c0e7580096.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 10:42:27 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             126W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   29C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   37C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:29143ms step_avg:nanms
step:2/1480 train_time:29254ms step_avg:nanms
step:3/1480 train_time:29371ms step_avg:nanms
step:4/1480 train_time:29511ms step_avg:nanms
step:5/1480 train_time:29653ms step_avg:nanms
step:6/1480 train_time:29794ms step_avg:nanms
step:7/1480 train_time:29939ms step_avg:nanms
step:8/1480 train_time:30078ms step_avg:nanms
step:9/1480 train_time:30220ms step_avg:nanms
step:10/1480 train_time:30364ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:282ms step_avg:nanms
step:13/1480 train_time:424ms step_avg:141.32ms
step:14/1480 train_time:566ms step_avg:141.47ms
step:15/1480 train_time:708ms step_avg:141.51ms
step:16/1480 train_time:851ms step_avg:141.88ms
step:17/1480 train_time:995ms step_avg:142.20ms
step:18/1480 train_time:1138ms step_avg:142.25ms
step:19/1480 train_time:1281ms step_avg:142.35ms
step:20/1480 train_time:1423ms step_avg:142.32ms
step:21/1480 train_time:1566ms step_avg:142.37ms
step:22/1480 train_time:1710ms step_avg:142.50ms
step:23/1480 train_time:1854ms step_avg:142.63ms
step:24/1480 train_time:2000ms step_avg:142.83ms
step:25/1480 train_time:2142ms step_avg:142.78ms
step:26/1480 train_time:2284ms step_avg:142.77ms
step:27/1480 train_time:2426ms step_avg:142.73ms
step:28/1480 train_time:2568ms step_avg:142.66ms
step:29/1480 train_time:2710ms step_avg:142.66ms
step:30/1480 train_time:2854ms step_avg:142.69ms
step:31/1480 train_time:2996ms step_avg:142.67ms
step:32/1480 train_time:3138ms step_avg:142.65ms
step:33/1480 train_time:3282ms step_avg:142.68ms
step:34/1480 train_time:3423ms step_avg:142.63ms
step:35/1480 train_time:3565ms step_avg:142.61ms
step:36/1480 train_time:3708ms step_avg:142.62ms
step:37/1480 train_time:3853ms step_avg:142.69ms
step:38/1480 train_time:3997ms step_avg:142.73ms
step:39/1480 train_time:4139ms step_avg:142.73ms
step:40/1480 train_time:4283ms step_avg:142.77ms
step:41/1480 train_time:4425ms step_avg:142.76ms
step:42/1480 train_time:4568ms step_avg:142.74ms
step:43/1480 train_time:4711ms step_avg:142.75ms
step:44/1480 train_time:4855ms step_avg:142.80ms
step:45/1480 train_time:4999ms step_avg:142.81ms
step:46/1480 train_time:5140ms step_avg:142.77ms
step:47/1480 train_time:5282ms step_avg:142.76ms
step:48/1480 train_time:5423ms step_avg:142.72ms
step:49/1480 train_time:5568ms step_avg:142.78ms
step:50/1480 train_time:5711ms step_avg:142.77ms
step:51/1480 train_time:5855ms step_avg:142.80ms
step:52/1480 train_time:5998ms step_avg:142.81ms
step:53/1480 train_time:6139ms step_avg:142.77ms
step:54/1480 train_time:6283ms step_avg:142.80ms
step:55/1480 train_time:6426ms step_avg:142.79ms
step:56/1480 train_time:6568ms step_avg:142.78ms
step:57/1480 train_time:6709ms step_avg:142.75ms
step:58/1480 train_time:6855ms step_avg:142.81ms
step:59/1480 train_time:6998ms step_avg:142.82ms
step:60/1480 train_time:7139ms step_avg:142.79ms
step:61/1480 train_time:7281ms step_avg:142.77ms
step:62/1480 train_time:7423ms step_avg:142.75ms
step:63/1480 train_time:7566ms step_avg:142.75ms
step:64/1480 train_time:7707ms step_avg:142.72ms
step:65/1480 train_time:7850ms step_avg:142.74ms
step:66/1480 train_time:7993ms step_avg:142.73ms
step:67/1480 train_time:8135ms step_avg:142.72ms
step:68/1480 train_time:8279ms step_avg:142.75ms
step:69/1480 train_time:8422ms step_avg:142.75ms
step:70/1480 train_time:8565ms step_avg:142.75ms
step:71/1480 train_time:8706ms step_avg:142.72ms
step:72/1480 train_time:8849ms step_avg:142.73ms
step:73/1480 train_time:8994ms step_avg:142.76ms
step:74/1480 train_time:9137ms step_avg:142.76ms
step:75/1480 train_time:9280ms step_avg:142.77ms
step:76/1480 train_time:9422ms step_avg:142.76ms
step:77/1480 train_time:9563ms step_avg:142.74ms
step:78/1480 train_time:9705ms step_avg:142.73ms
step:79/1480 train_time:9849ms step_avg:142.74ms
step:80/1480 train_time:10374ms step_avg:148.20ms
step:81/1480 train_time:10991ms step_avg:154.80ms
step:82/1480 train_time:11090ms step_avg:154.03ms
step:83/1480 train_time:11232ms step_avg:153.87ms
step:84/1480 train_time:11375ms step_avg:153.72ms
step:85/1480 train_time:11517ms step_avg:153.56ms
step:86/1480 train_time:11659ms step_avg:153.41ms
step:87/1480 train_time:11802ms step_avg:153.27ms
step:88/1480 train_time:11943ms step_avg:153.11ms
step:89/1480 train_time:12086ms step_avg:152.98ms
step:90/1480 train_time:12229ms step_avg:152.86ms
step:91/1480 train_time:12373ms step_avg:152.75ms
step:92/1480 train_time:12515ms step_avg:152.63ms
step:93/1480 train_time:12658ms step_avg:152.50ms
step:94/1480 train_time:12800ms step_avg:152.38ms
step:95/1480 train_time:12942ms step_avg:152.25ms
step:96/1480 train_time:13461ms step_avg:156.52ms
step:97/1480 train_time:13563ms step_avg:155.90ms
step:98/1480 train_time:13706ms step_avg:155.75ms
step:99/1480 train_time:13848ms step_avg:155.59ms
step:100/1480 train_time:13991ms step_avg:155.46ms
step:101/1480 train_time:14136ms step_avg:155.34ms
step:102/1480 train_time:14276ms step_avg:155.17ms
step:103/1480 train_time:14418ms step_avg:155.03ms
step:104/1480 train_time:14562ms step_avg:154.91ms
step:105/1480 train_time:14703ms step_avg:154.77ms
step:106/1480 train_time:14846ms step_avg:154.64ms
step:107/1480 train_time:14988ms step_avg:154.52ms
step:108/1480 train_time:15133ms step_avg:154.42ms
step:109/1480 train_time:15277ms step_avg:154.31ms
step:110/1480 train_time:15420ms step_avg:154.20ms
step:111/1480 train_time:15563ms step_avg:154.09ms
step:112/1480 train_time:15707ms step_avg:153.99ms
step:113/1480 train_time:15852ms step_avg:153.90ms
step:114/1480 train_time:15998ms step_avg:153.83ms
step:115/1480 train_time:16143ms step_avg:153.74ms
step:116/1480 train_time:16289ms step_avg:153.67ms
step:117/1480 train_time:16436ms step_avg:153.61ms
step:118/1480 train_time:16582ms step_avg:153.54ms
step:119/1480 train_time:16727ms step_avg:153.46ms
step:120/1480 train_time:16872ms step_avg:153.39ms
step:121/1480 train_time:17019ms step_avg:153.32ms
step:122/1480 train_time:17165ms step_avg:153.26ms
step:123/1480 train_time:17309ms step_avg:153.18ms
step:124/1480 train_time:17456ms step_avg:153.12ms
step:125/1480 train_time:17602ms step_avg:153.06ms
step:125/1480 val_loss:4.3847 train_time:17666ms step_avg:153.62ms
step:126/1480 train_time:17761ms step_avg:153.11ms
step:127/1480 train_time:17906ms step_avg:153.04ms
step:128/1480 train_time:18053ms step_avg:152.99ms
step:129/1480 train_time:18198ms step_avg:152.92ms
step:130/1480 train_time:18343ms step_avg:152.85ms
step:131/1480 train_time:18489ms step_avg:152.80ms
step:132/1480 train_time:18635ms step_avg:152.74ms
step:133/1480 train_time:18779ms step_avg:152.67ms
step:134/1480 train_time:18926ms step_avg:152.63ms
step:135/1480 train_time:19073ms step_avg:152.59ms
step:136/1480 train_time:19222ms step_avg:152.55ms
step:137/1480 train_time:19367ms step_avg:152.50ms
step:138/1480 train_time:19514ms step_avg:152.45ms
step:139/1480 train_time:19659ms step_avg:152.39ms
step:140/1480 train_time:19804ms step_avg:152.34ms
step:141/1480 train_time:19952ms step_avg:152.30ms
step:142/1480 train_time:20097ms step_avg:152.25ms
step:143/1480 train_time:20242ms step_avg:152.20ms
step:144/1480 train_time:20390ms step_avg:152.16ms
step:145/1480 train_time:20536ms step_avg:152.12ms
step:146/1480 train_time:20681ms step_avg:152.07ms
step:147/1480 train_time:20828ms step_avg:152.03ms
step:148/1480 train_time:20974ms step_avg:151.99ms
step:149/1480 train_time:21119ms step_avg:151.94ms
step:150/1480 train_time:21265ms step_avg:151.89ms
step:151/1480 train_time:21412ms step_avg:151.86ms
step:152/1480 train_time:21557ms step_avg:151.81ms
step:153/1480 train_time:21702ms step_avg:151.76ms
step:154/1480 train_time:21849ms step_avg:151.73ms
step:155/1480 train_time:21995ms step_avg:151.69ms
step:156/1480 train_time:22140ms step_avg:151.64ms
step:157/1480 train_time:22287ms step_avg:151.61ms
step:158/1480 train_time:22434ms step_avg:151.58ms
step:159/1480 train_time:22579ms step_avg:151.54ms
step:160/1480 train_time:22725ms step_avg:151.50ms
step:161/1480 train_time:22871ms step_avg:151.46ms
step:162/1480 train_time:23016ms step_avg:151.42ms
step:163/1480 train_time:23161ms step_avg:151.38ms
step:164/1480 train_time:23308ms step_avg:151.35ms
step:165/1480 train_time:23454ms step_avg:151.32ms
step:166/1480 train_time:23600ms step_avg:151.28ms
step:167/1480 train_time:23745ms step_avg:151.24ms
step:168/1480 train_time:23891ms step_avg:151.21ms
step:169/1480 train_time:24036ms step_avg:151.17ms
step:170/1480 train_time:24181ms step_avg:151.13ms
step:171/1480 train_time:24329ms step_avg:151.11ms
step:172/1480 train_time:24475ms step_avg:151.08ms
step:173/1480 train_time:24621ms step_avg:151.05ms
step:174/1480 train_time:24767ms step_avg:151.02ms
step:175/1480 train_time:24914ms step_avg:150.99ms
step:176/1480 train_time:25058ms step_avg:150.95ms
step:177/1480 train_time:25203ms step_avg:150.92ms
step:178/1480 train_time:25350ms step_avg:150.89ms
step:179/1480 train_time:25495ms step_avg:150.86ms
step:180/1480 train_time:25641ms step_avg:150.83ms
step:181/1480 train_time:25786ms step_avg:150.80ms
step:182/1480 train_time:25933ms step_avg:150.77ms
step:183/1480 train_time:26077ms step_avg:150.73ms
step:184/1480 train_time:26223ms step_avg:150.71ms
step:185/1480 train_time:26370ms step_avg:150.69ms
step:186/1480 train_time:26516ms step_avg:150.66ms
step:187/1480 train_time:26661ms step_avg:150.63ms
step:188/1480 train_time:26807ms step_avg:150.60ms
step:189/1480 train_time:26977ms step_avg:150.71ms
step:190/1480 train_time:27098ms step_avg:150.54ms
step:191/1480 train_time:27243ms step_avg:150.52ms
step:192/1480 train_time:27390ms step_avg:150.49ms
step:193/1480 train_time:27536ms step_avg:150.47ms
step:194/1480 train_time:27680ms step_avg:150.44ms
step:195/1480 train_time:27828ms step_avg:150.42ms
step:196/1480 train_time:27974ms step_avg:150.40ms
step:197/1480 train_time:28119ms step_avg:150.37ms
step:198/1480 train_time:28264ms step_avg:150.34ms
step:199/1480 train_time:28411ms step_avg:150.32ms
step:200/1480 train_time:28557ms step_avg:150.30ms
step:201/1480 train_time:28703ms step_avg:150.28ms
step:202/1480 train_time:28848ms step_avg:150.25ms
step:203/1480 train_time:28994ms step_avg:150.23ms
step:204/1480 train_time:29139ms step_avg:150.20ms
step:205/1480 train_time:29284ms step_avg:150.18ms
step:206/1480 train_time:29431ms step_avg:150.16ms
step:207/1480 train_time:29576ms step_avg:150.13ms
step:208/1480 train_time:29721ms step_avg:150.11ms
step:209/1480 train_time:29868ms step_avg:150.09ms
step:210/1480 train_time:30014ms step_avg:150.07ms
step:211/1480 train_time:30158ms step_avg:150.04ms
step:212/1480 train_time:30304ms step_avg:150.02ms
step:213/1480 train_time:30450ms step_avg:150.00ms
step:214/1480 train_time:30597ms step_avg:149.98ms
step:215/1480 train_time:30742ms step_avg:149.96ms
step:216/1480 train_time:30889ms step_avg:149.94ms
step:217/1480 train_time:31035ms step_avg:149.93ms
step:218/1480 train_time:31563ms step_avg:151.74ms
step:219/1480 train_time:31667ms step_avg:151.52ms
step:220/1480 train_time:31813ms step_avg:151.49ms
step:221/1480 train_time:32384ms step_avg:153.48ms
step:222/1480 train_time:32490ms step_avg:153.25ms
step:223/1480 train_time:32639ms step_avg:153.24ms
step:224/1480 train_time:32787ms step_avg:153.21ms
step:225/1480 train_time:32936ms step_avg:153.19ms
step:226/1480 train_time:33083ms step_avg:153.16ms
step:227/1480 train_time:33232ms step_avg:153.14ms
step:228/1480 train_time:33381ms step_avg:153.12ms
step:229/1480 train_time:33531ms step_avg:153.11ms
step:230/1480 train_time:33679ms step_avg:153.09ms
step:231/1480 train_time:33828ms step_avg:153.07ms
step:232/1480 train_time:33977ms step_avg:153.05ms
step:233/1480 train_time:34126ms step_avg:153.03ms
step:234/1480 train_time:34275ms step_avg:153.01ms
step:235/1480 train_time:34424ms step_avg:152.99ms
step:236/1480 train_time:34573ms step_avg:152.98ms
step:237/1480 train_time:34723ms step_avg:152.97ms
step:238/1480 train_time:34872ms step_avg:152.95ms
step:239/1480 train_time:35020ms step_avg:152.93ms
step:240/1480 train_time:35170ms step_avg:152.91ms
step:241/1480 train_time:35318ms step_avg:152.89ms
step:242/1480 train_time:35466ms step_avg:152.87ms
step:243/1480 train_time:35615ms step_avg:152.85ms
step:244/1480 train_time:35762ms step_avg:152.83ms
step:245/1480 train_time:35912ms step_avg:152.82ms
step:246/1480 train_time:36059ms step_avg:152.79ms
step:247/1480 train_time:36209ms step_avg:152.78ms
step:248/1480 train_time:36357ms step_avg:152.76ms
step:249/1480 train_time:36506ms step_avg:152.74ms
step:250/1480 train_time:36655ms step_avg:152.73ms
step:250/1480 val_loss:3.9817 train_time:36721ms step_avg:153.00ms
step:251/1480 train_time:36815ms step_avg:152.76ms
step:252/1480 train_time:36959ms step_avg:152.72ms
step:253/1480 train_time:37108ms step_avg:152.71ms
step:254/1480 train_time:37255ms step_avg:152.69ms
step:255/1480 train_time:37404ms step_avg:152.67ms
step:256/1480 train_time:37551ms step_avg:152.64ms
step:257/1480 train_time:37699ms step_avg:152.63ms
step:258/1480 train_time:37848ms step_avg:152.61ms
step:259/1480 train_time:37998ms step_avg:152.60ms
step:260/1480 train_time:38147ms step_avg:152.59ms
step:261/1480 train_time:38294ms step_avg:152.56ms
step:262/1480 train_time:38444ms step_avg:152.55ms
step:263/1480 train_time:38591ms step_avg:152.53ms
step:264/1480 train_time:38740ms step_avg:152.52ms
step:265/1480 train_time:38888ms step_avg:152.50ms
step:266/1480 train_time:39037ms step_avg:152.49ms
step:267/1480 train_time:39185ms step_avg:152.47ms
step:268/1480 train_time:39334ms step_avg:152.46ms
step:269/1480 train_time:39482ms step_avg:152.44ms
step:270/1480 train_time:39631ms step_avg:152.43ms
step:271/1480 train_time:39780ms step_avg:152.41ms
step:272/1480 train_time:39928ms step_avg:152.40ms
step:273/1480 train_time:40076ms step_avg:152.38ms
step:274/1480 train_time:40225ms step_avg:152.37ms
step:275/1480 train_time:40372ms step_avg:152.35ms
step:276/1480 train_time:40522ms step_avg:152.34ms
step:277/1480 train_time:40669ms step_avg:152.32ms
step:278/1480 train_time:40818ms step_avg:152.31ms
step:279/1480 train_time:40967ms step_avg:152.29ms
step:280/1480 train_time:41115ms step_avg:152.28ms
step:281/1480 train_time:41264ms step_avg:152.27ms
step:282/1480 train_time:41413ms step_avg:152.25ms
step:283/1480 train_time:41562ms step_avg:152.24ms
step:284/1480 train_time:41710ms step_avg:152.23ms
step:285/1480 train_time:41859ms step_avg:152.22ms
step:286/1480 train_time:42007ms step_avg:152.20ms
step:287/1480 train_time:42156ms step_avg:152.19ms
step:288/1480 train_time:42305ms step_avg:152.18ms
step:289/1480 train_time:42453ms step_avg:152.16ms
step:290/1480 train_time:42601ms step_avg:152.15ms
step:291/1480 train_time:42749ms step_avg:152.13ms
step:292/1480 train_time:42898ms step_avg:152.12ms
step:293/1480 train_time:43047ms step_avg:152.11ms
step:294/1480 train_time:43195ms step_avg:152.09ms
step:295/1480 train_time:43344ms step_avg:152.09ms
step:296/1480 train_time:43492ms step_avg:152.07ms
step:297/1480 train_time:43642ms step_avg:152.06ms
step:298/1480 train_time:43789ms step_avg:152.05ms
step:299/1480 train_time:43939ms step_avg:152.04ms
step:300/1480 train_time:44089ms step_avg:152.03ms
step:301/1480 train_time:44238ms step_avg:152.02ms
step:302/1480 train_time:44386ms step_avg:152.01ms
step:303/1480 train_time:44535ms step_avg:152.00ms
step:304/1480 train_time:44683ms step_avg:151.98ms
step:305/1480 train_time:44833ms step_avg:151.98ms
step:306/1480 train_time:44982ms step_avg:151.97ms
step:307/1480 train_time:45131ms step_avg:151.96ms
step:308/1480 train_time:45280ms step_avg:151.95ms
step:309/1480 train_time:45428ms step_avg:151.93ms
step:310/1480 train_time:45577ms step_avg:151.92ms
step:311/1480 train_time:45726ms step_avg:151.91ms
step:312/1480 train_time:45873ms step_avg:151.90ms
step:313/1480 train_time:46023ms step_avg:151.89ms
step:314/1480 train_time:46170ms step_avg:151.88ms
step:315/1480 train_time:46320ms step_avg:151.87ms
step:316/1480 train_time:46468ms step_avg:151.86ms
step:317/1480 train_time:46617ms step_avg:151.85ms
step:318/1480 train_time:46766ms step_avg:151.84ms
step:319/1480 train_time:46915ms step_avg:151.83ms
step:320/1480 train_time:47065ms step_avg:151.82ms
step:321/1480 train_time:47212ms step_avg:151.81ms
step:322/1480 train_time:47360ms step_avg:151.80ms
step:323/1480 train_time:47508ms step_avg:151.78ms
step:324/1480 train_time:47656ms step_avg:151.77ms
step:325/1480 train_time:47805ms step_avg:151.76ms
step:326/1480 train_time:47954ms step_avg:151.75ms
step:327/1480 train_time:48104ms step_avg:151.75ms
step:328/1480 train_time:48252ms step_avg:151.73ms
step:329/1480 train_time:48401ms step_avg:151.73ms
step:330/1480 train_time:48551ms step_avg:151.72ms
step:331/1480 train_time:48702ms step_avg:151.72ms
step:332/1480 train_time:48852ms step_avg:151.71ms
step:333/1480 train_time:49003ms step_avg:151.71ms
step:334/1480 train_time:49153ms step_avg:151.71ms
step:335/1480 train_time:49304ms step_avg:151.71ms
step:336/1480 train_time:49455ms step_avg:151.70ms
step:337/1480 train_time:49606ms step_avg:151.70ms
step:338/1480 train_time:49756ms step_avg:151.69ms
step:339/1480 train_time:49907ms step_avg:151.69ms
step:340/1480 train_time:50057ms step_avg:151.69ms
step:341/1480 train_time:50208ms step_avg:151.69ms
step:342/1480 train_time:50360ms step_avg:151.69ms
step:343/1480 train_time:50510ms step_avg:151.68ms
step:344/1480 train_time:50661ms step_avg:151.68ms
step:345/1480 train_time:50811ms step_avg:151.67ms
step:346/1480 train_time:50963ms step_avg:151.67ms
step:347/1480 train_time:51114ms step_avg:151.67ms
step:348/1480 train_time:51266ms step_avg:151.67ms
step:349/1480 train_time:51416ms step_avg:151.67ms
step:350/1480 train_time:51567ms step_avg:151.67ms
step:351/1480 train_time:51719ms step_avg:151.67ms
step:352/1480 train_time:51870ms step_avg:151.67ms
step:353/1480 train_time:52022ms step_avg:151.67ms
step:354/1480 train_time:52172ms step_avg:151.66ms
step:355/1480 train_time:52324ms step_avg:151.66ms
step:356/1480 train_time:52474ms step_avg:151.66ms
step:357/1480 train_time:52625ms step_avg:151.66ms
step:358/1480 train_time:52775ms step_avg:151.65ms
step:359/1480 train_time:52927ms step_avg:151.65ms
step:360/1480 train_time:53077ms step_avg:151.65ms
step:361/1480 train_time:53229ms step_avg:151.65ms
step:362/1480 train_time:53380ms step_avg:151.65ms
step:363/1480 train_time:53531ms step_avg:151.65ms
step:364/1480 train_time:53684ms step_avg:151.65ms
step:365/1480 train_time:53834ms step_avg:151.64ms
step:366/1480 train_time:53984ms step_avg:151.64ms
step:367/1480 train_time:54136ms step_avg:151.64ms
step:368/1480 train_time:54288ms step_avg:151.64ms
step:369/1480 train_time:54439ms step_avg:151.64ms
step:370/1480 train_time:54590ms step_avg:151.64ms
step:371/1480 train_time:54740ms step_avg:151.64ms
step:372/1480 train_time:54891ms step_avg:151.63ms
step:373/1480 train_time:55043ms step_avg:151.63ms
step:374/1480 train_time:55193ms step_avg:151.63ms
step:375/1480 train_time:55345ms step_avg:151.63ms
step:375/1480 val_loss:3.8022 train_time:55412ms step_avg:151.81ms
step:376/1480 train_time:55503ms step_avg:151.65ms
step:377/1480 train_time:55654ms step_avg:151.64ms
step:378/1480 train_time:55804ms step_avg:151.64ms
step:379/1480 train_time:55976ms step_avg:151.70ms
step:380/1480 train_time:56105ms step_avg:151.63ms
step:381/1480 train_time:56254ms step_avg:151.63ms
step:382/1480 train_time:56406ms step_avg:151.63ms
step:383/1480 train_time:56557ms step_avg:151.63ms
step:384/1480 train_time:56709ms step_avg:151.63ms
step:385/1480 train_time:56859ms step_avg:151.62ms
step:386/1480 train_time:57010ms step_avg:151.62ms
step:387/1480 train_time:57161ms step_avg:151.62ms
step:388/1480 train_time:57311ms step_avg:151.62ms
step:389/1480 train_time:57464ms step_avg:151.62ms
step:390/1480 train_time:57614ms step_avg:151.62ms
step:391/1480 train_time:57766ms step_avg:151.62ms
step:392/1480 train_time:57917ms step_avg:151.62ms
step:393/1480 train_time:58068ms step_avg:151.61ms
step:394/1480 train_time:58219ms step_avg:151.61ms
step:395/1480 train_time:58369ms step_avg:151.61ms
step:396/1480 train_time:58521ms step_avg:151.61ms
step:397/1480 train_time:58671ms step_avg:151.60ms
step:398/1480 train_time:58824ms step_avg:151.61ms
step:399/1480 train_time:58975ms step_avg:151.61ms
step:400/1480 train_time:59127ms step_avg:151.61ms
step:401/1480 train_time:59278ms step_avg:151.61ms
step:402/1480 train_time:59429ms step_avg:151.60ms
step:403/1480 train_time:59579ms step_avg:151.60ms
step:404/1480 train_time:59730ms step_avg:151.60ms
step:405/1480 train_time:59882ms step_avg:151.60ms
step:406/1480 train_time:60033ms step_avg:151.60ms
step:407/1480 train_time:60185ms step_avg:151.60ms
step:408/1480 train_time:60336ms step_avg:151.60ms
step:409/1480 train_time:60487ms step_avg:151.60ms
step:410/1480 train_time:60638ms step_avg:151.59ms
step:411/1480 train_time:60789ms step_avg:151.59ms
step:412/1480 train_time:60940ms step_avg:151.59ms
step:413/1480 train_time:61091ms step_avg:151.59ms
step:414/1480 train_time:61243ms step_avg:151.59ms
step:415/1480 train_time:61394ms step_avg:151.59ms
step:416/1480 train_time:61546ms step_avg:151.59ms
step:417/1480 train_time:61696ms step_avg:151.59ms
step:418/1480 train_time:61847ms step_avg:151.59ms
step:419/1480 train_time:61997ms step_avg:151.58ms
step:420/1480 train_time:62148ms step_avg:151.58ms
step:421/1480 train_time:62299ms step_avg:151.58ms
step:422/1480 train_time:62450ms step_avg:151.58ms
step:423/1480 train_time:62601ms step_avg:151.58ms
step:424/1480 train_time:62752ms step_avg:151.57ms
step:425/1480 train_time:62904ms step_avg:151.58ms
step:426/1480 train_time:63054ms step_avg:151.57ms
step:427/1480 train_time:63205ms step_avg:151.57ms
step:428/1480 train_time:63355ms step_avg:151.57ms
step:429/1480 train_time:63506ms step_avg:151.57ms
step:430/1480 train_time:63657ms step_avg:151.56ms
step:431/1480 train_time:63808ms step_avg:151.56ms
step:432/1480 train_time:63960ms step_avg:151.56ms
step:433/1480 train_time:64111ms step_avg:151.56ms
step:434/1480 train_time:64262ms step_avg:151.56ms
step:435/1480 train_time:64412ms step_avg:151.56ms
step:436/1480 train_time:64564ms step_avg:151.56ms
step:437/1480 train_time:64714ms step_avg:151.56ms
step:438/1480 train_time:64866ms step_avg:151.56ms
step:439/1480 train_time:65017ms step_avg:151.55ms
step:440/1480 train_time:65170ms step_avg:151.56ms
step:441/1480 train_time:65323ms step_avg:151.56ms
step:442/1480 train_time:65476ms step_avg:151.56ms
step:443/1480 train_time:65629ms step_avg:151.57ms
step:444/1480 train_time:65781ms step_avg:151.57ms
step:445/1480 train_time:65933ms step_avg:151.57ms
step:446/1480 train_time:66086ms step_avg:151.57ms
step:447/1480 train_time:66241ms step_avg:151.58ms
step:448/1480 train_time:66394ms step_avg:151.58ms
step:449/1480 train_time:66548ms step_avg:151.59ms
step:450/1480 train_time:66700ms step_avg:151.59ms
step:451/1480 train_time:66853ms step_avg:151.59ms
step:452/1480 train_time:67005ms step_avg:151.60ms
step:453/1480 train_time:67158ms step_avg:151.60ms
step:454/1480 train_time:67312ms step_avg:151.60ms
step:455/1480 train_time:67466ms step_avg:151.61ms
step:456/1480 train_time:67619ms step_avg:151.61ms
step:457/1480 train_time:67773ms step_avg:151.62ms
step:458/1480 train_time:67926ms step_avg:151.62ms
step:459/1480 train_time:68079ms step_avg:151.62ms
step:460/1480 train_time:68232ms step_avg:151.63ms
step:461/1480 train_time:68385ms step_avg:151.63ms
step:462/1480 train_time:68540ms step_avg:151.64ms
step:463/1480 train_time:68694ms step_avg:151.64ms
step:464/1480 train_time:68848ms step_avg:151.65ms
step:465/1480 train_time:69000ms step_avg:151.65ms
step:466/1480 train_time:69152ms step_avg:151.65ms
step:467/1480 train_time:69306ms step_avg:151.65ms
step:468/1480 train_time:69458ms step_avg:151.66ms
step:469/1480 train_time:69611ms step_avg:151.66ms
step:470/1480 train_time:69765ms step_avg:151.66ms
step:471/1480 train_time:69917ms step_avg:151.66ms
step:472/1480 train_time:70070ms step_avg:151.67ms
step:473/1480 train_time:70223ms step_avg:151.67ms
step:474/1480 train_time:70376ms step_avg:151.67ms
step:475/1480 train_time:70529ms step_avg:151.67ms
step:476/1480 train_time:70682ms step_avg:151.68ms
step:477/1480 train_time:70835ms step_avg:151.68ms
step:478/1480 train_time:70988ms step_avg:151.68ms
step:479/1480 train_time:71141ms step_avg:151.69ms
step:480/1480 train_time:71294ms step_avg:151.69ms
step:481/1480 train_time:71448ms step_avg:151.69ms
step:482/1480 train_time:71601ms step_avg:151.70ms
step:483/1480 train_time:71754ms step_avg:151.70ms
step:484/1480 train_time:71908ms step_avg:151.70ms
step:485/1480 train_time:72060ms step_avg:151.71ms
step:486/1480 train_time:72213ms step_avg:151.71ms
step:487/1480 train_time:72366ms step_avg:151.71ms
step:488/1480 train_time:72521ms step_avg:151.72ms
step:489/1480 train_time:72673ms step_avg:151.72ms
step:490/1480 train_time:72827ms step_avg:151.72ms
step:491/1480 train_time:72979ms step_avg:151.72ms
step:492/1480 train_time:73132ms step_avg:151.73ms
step:493/1480 train_time:73285ms step_avg:151.73ms
step:494/1480 train_time:73438ms step_avg:151.73ms
step:495/1480 train_time:73591ms step_avg:151.73ms
step:496/1480 train_time:73745ms step_avg:151.74ms
step:497/1480 train_time:73898ms step_avg:151.74ms
step:498/1480 train_time:74050ms step_avg:151.74ms
step:499/1480 train_time:74203ms step_avg:151.74ms
step:500/1480 train_time:74355ms step_avg:151.74ms
step:500/1480 val_loss:3.6858 train_time:74424ms step_avg:151.88ms
step:501/1480 train_time:74514ms step_avg:151.76ms
step:502/1480 train_time:74666ms step_avg:151.76ms
step:503/1480 train_time:74819ms step_avg:151.76ms
step:504/1480 train_time:74971ms step_avg:151.76ms
step:505/1480 train_time:75124ms step_avg:151.77ms
step:506/1480 train_time:75277ms step_avg:151.77ms
step:507/1480 train_time:75430ms step_avg:151.77ms
step:508/1480 train_time:75584ms step_avg:151.78ms
step:509/1480 train_time:75738ms step_avg:151.78ms
step:510/1480 train_time:75892ms step_avg:151.78ms
step:511/1480 train_time:76045ms step_avg:151.79ms
step:512/1480 train_time:76197ms step_avg:151.79ms
step:513/1480 train_time:76351ms step_avg:151.79ms
step:514/1480 train_time:76503ms step_avg:151.79ms
step:515/1480 train_time:76658ms step_avg:151.80ms
step:516/1480 train_time:76813ms step_avg:151.80ms
step:517/1480 train_time:76965ms step_avg:151.81ms
step:518/1480 train_time:77119ms step_avg:151.81ms
step:519/1480 train_time:77272ms step_avg:151.81ms
step:520/1480 train_time:77425ms step_avg:151.81ms
step:521/1480 train_time:77579ms step_avg:151.82ms
step:522/1480 train_time:77734ms step_avg:151.82ms
step:523/1480 train_time:77887ms step_avg:151.83ms
step:524/1480 train_time:78040ms step_avg:151.83ms
step:525/1480 train_time:78193ms step_avg:151.83ms
step:526/1480 train_time:78346ms step_avg:151.83ms
step:527/1480 train_time:78499ms step_avg:151.83ms
step:528/1480 train_time:78652ms step_avg:151.84ms
step:529/1480 train_time:78805ms step_avg:151.84ms
step:530/1480 train_time:78959ms step_avg:151.84ms
step:531/1480 train_time:79113ms step_avg:151.85ms
step:532/1480 train_time:79266ms step_avg:151.85ms
step:533/1480 train_time:79419ms step_avg:151.85ms
step:534/1480 train_time:79572ms step_avg:151.86ms
step:535/1480 train_time:79727ms step_avg:151.86ms
step:536/1480 train_time:79881ms step_avg:151.86ms
step:537/1480 train_time:80034ms step_avg:151.87ms
step:538/1480 train_time:80187ms step_avg:151.87ms
step:539/1480 train_time:80340ms step_avg:151.87ms
step:540/1480 train_time:80495ms step_avg:151.88ms
step:541/1480 train_time:80648ms step_avg:151.88ms
step:542/1480 train_time:80800ms step_avg:151.88ms
step:543/1480 train_time:80954ms step_avg:151.88ms
step:544/1480 train_time:81107ms step_avg:151.89ms
step:545/1480 train_time:81260ms step_avg:151.89ms
step:546/1480 train_time:81413ms step_avg:151.89ms
step:547/1480 train_time:81565ms step_avg:151.89ms
step:548/1480 train_time:81717ms step_avg:151.89ms
step:549/1480 train_time:81870ms step_avg:151.89ms
step:550/1480 train_time:82025ms step_avg:151.90ms
step:551/1480 train_time:82180ms step_avg:151.90ms
step:552/1480 train_time:82335ms step_avg:151.91ms
step:553/1480 train_time:82490ms step_avg:151.92ms
step:554/1480 train_time:82644ms step_avg:151.92ms
step:555/1480 train_time:82798ms step_avg:151.92ms
step:556/1480 train_time:82954ms step_avg:151.93ms
step:557/1480 train_time:83110ms step_avg:151.94ms
step:558/1480 train_time:83265ms step_avg:151.94ms
step:559/1480 train_time:83420ms step_avg:151.95ms
step:560/1480 train_time:83574ms step_avg:151.95ms
step:561/1480 train_time:83729ms step_avg:151.96ms
step:562/1480 train_time:83883ms step_avg:151.96ms
step:563/1480 train_time:84038ms step_avg:151.97ms
step:564/1480 train_time:84194ms step_avg:151.97ms
step:565/1480 train_time:84349ms step_avg:151.98ms
step:566/1480 train_time:84504ms step_avg:151.99ms
step:567/1480 train_time:84660ms step_avg:151.99ms
step:568/1480 train_time:84814ms step_avg:152.00ms
step:569/1480 train_time:84983ms step_avg:152.03ms
step:570/1480 train_time:85125ms step_avg:152.01ms
step:571/1480 train_time:85280ms step_avg:152.01ms
step:572/1480 train_time:85434ms step_avg:152.02ms
step:573/1480 train_time:85589ms step_avg:152.02ms
step:574/1480 train_time:85746ms step_avg:152.03ms
step:575/1480 train_time:85901ms step_avg:152.04ms
step:576/1480 train_time:86056ms step_avg:152.04ms
step:577/1480 train_time:86210ms step_avg:152.05ms
step:578/1480 train_time:86365ms step_avg:152.05ms
step:579/1480 train_time:86519ms step_avg:152.05ms
step:580/1480 train_time:86673ms step_avg:152.06ms
step:581/1480 train_time:86830ms step_avg:152.07ms
step:582/1480 train_time:86985ms step_avg:152.07ms
step:583/1480 train_time:87140ms step_avg:152.08ms
step:584/1480 train_time:87295ms step_avg:152.08ms
step:585/1480 train_time:87450ms step_avg:152.09ms
step:586/1480 train_time:87605ms step_avg:152.09ms
step:587/1480 train_time:87761ms step_avg:152.10ms
step:588/1480 train_time:87915ms step_avg:152.10ms
step:589/1480 train_time:88069ms step_avg:152.11ms
step:590/1480 train_time:88226ms step_avg:152.11ms
step:591/1480 train_time:88381ms step_avg:152.12ms
step:592/1480 train_time:88536ms step_avg:152.12ms
step:593/1480 train_time:88691ms step_avg:152.13ms
step:594/1480 train_time:88846ms step_avg:152.13ms
step:595/1480 train_time:89003ms step_avg:152.14ms
step:596/1480 train_time:89160ms step_avg:152.15ms
step:597/1480 train_time:89315ms step_avg:152.15ms
step:598/1480 train_time:89468ms step_avg:152.16ms
step:599/1480 train_time:89622ms step_avg:152.16ms
step:600/1480 train_time:89778ms step_avg:152.17ms
step:601/1480 train_time:89935ms step_avg:152.17ms
step:602/1480 train_time:90090ms step_avg:152.18ms
step:603/1480 train_time:90245ms step_avg:152.18ms
step:604/1480 train_time:90401ms step_avg:152.19ms
step:605/1480 train_time:90555ms step_avg:152.19ms
step:606/1480 train_time:90710ms step_avg:152.20ms
step:607/1480 train_time:90866ms step_avg:152.20ms
step:608/1480 train_time:91022ms step_avg:152.21ms
step:609/1480 train_time:91177ms step_avg:152.21ms
step:610/1480 train_time:91333ms step_avg:152.22ms
step:611/1480 train_time:91488ms step_avg:152.23ms
step:612/1480 train_time:91642ms step_avg:152.23ms
step:613/1480 train_time:91798ms step_avg:152.24ms
step:614/1480 train_time:91953ms step_avg:152.24ms
step:615/1480 train_time:92107ms step_avg:152.24ms
step:616/1480 train_time:92262ms step_avg:152.25ms
step:617/1480 train_time:92416ms step_avg:152.25ms
step:618/1480 train_time:92570ms step_avg:152.25ms
step:619/1480 train_time:92726ms step_avg:152.26ms
step:620/1480 train_time:92881ms step_avg:152.26ms
step:621/1480 train_time:93036ms step_avg:152.27ms
step:622/1480 train_time:93190ms step_avg:152.27ms
step:623/1480 train_time:93345ms step_avg:152.28ms
step:624/1480 train_time:93501ms step_avg:152.28ms
step:625/1480 train_time:93655ms step_avg:152.28ms
step:625/1480 val_loss:3.6036 train_time:93726ms step_avg:152.40ms
step:626/1480 train_time:93817ms step_avg:152.30ms
step:627/1480 train_time:93970ms step_avg:152.30ms
step:628/1480 train_time:94125ms step_avg:152.31ms
step:629/1480 train_time:94280ms step_avg:152.31ms
step:630/1480 train_time:94434ms step_avg:152.31ms
step:631/1480 train_time:94589ms step_avg:152.32ms
step:632/1480 train_time:94743ms step_avg:152.32ms
step:633/1480 train_time:94899ms step_avg:152.33ms
step:634/1480 train_time:95055ms step_avg:152.33ms
step:635/1480 train_time:95209ms step_avg:152.33ms
step:636/1480 train_time:95362ms step_avg:152.34ms
step:637/1480 train_time:95517ms step_avg:152.34ms
step:638/1480 train_time:95672ms step_avg:152.34ms
step:639/1480 train_time:95827ms step_avg:152.35ms
step:640/1480 train_time:95983ms step_avg:152.35ms
step:641/1480 train_time:96138ms step_avg:152.36ms
step:642/1480 train_time:96292ms step_avg:152.36ms
step:643/1480 train_time:96446ms step_avg:152.36ms
step:644/1480 train_time:96602ms step_avg:152.37ms
step:645/1480 train_time:96757ms step_avg:152.37ms
step:646/1480 train_time:96913ms step_avg:152.38ms
step:647/1480 train_time:97068ms step_avg:152.38ms
step:648/1480 train_time:97226ms step_avg:152.39ms
step:649/1480 train_time:97381ms step_avg:152.40ms
step:650/1480 train_time:97535ms step_avg:152.40ms
step:651/1480 train_time:97690ms step_avg:152.40ms
step:652/1480 train_time:97844ms step_avg:152.40ms
step:653/1480 train_time:98000ms step_avg:152.41ms
step:654/1480 train_time:98155ms step_avg:152.41ms
step:655/1480 train_time:98310ms step_avg:152.42ms
step:656/1480 train_time:98464ms step_avg:152.42ms
step:657/1480 train_time:98619ms step_avg:152.42ms
step:658/1480 train_time:98774ms step_avg:152.43ms
step:659/1480 train_time:98929ms step_avg:152.43ms
step:660/1480 train_time:99085ms step_avg:152.44ms
step:661/1480 train_time:99242ms step_avg:152.45ms
step:662/1480 train_time:99400ms step_avg:152.45ms
step:663/1480 train_time:99557ms step_avg:152.46ms
step:664/1480 train_time:99712ms step_avg:152.46ms
step:665/1480 train_time:99869ms step_avg:152.47ms
step:666/1480 train_time:100024ms step_avg:152.48ms
step:667/1480 train_time:100182ms step_avg:152.48ms
step:668/1480 train_time:100339ms step_avg:152.49ms
step:669/1480 train_time:100496ms step_avg:152.50ms
step:670/1480 train_time:100652ms step_avg:152.50ms
step:671/1480 train_time:100810ms step_avg:152.51ms
step:672/1480 train_time:100967ms step_avg:152.52ms
step:673/1480 train_time:101123ms step_avg:152.52ms
step:674/1480 train_time:101280ms step_avg:152.53ms
step:675/1480 train_time:101437ms step_avg:152.54ms
step:676/1480 train_time:101595ms step_avg:152.54ms
step:677/1480 train_time:101751ms step_avg:152.55ms
step:678/1480 train_time:101908ms step_avg:152.56ms
step:679/1480 train_time:102065ms step_avg:152.56ms
step:680/1480 train_time:102222ms step_avg:152.57ms
step:681/1480 train_time:102378ms step_avg:152.58ms
step:682/1480 train_time:102535ms step_avg:152.58ms
step:683/1480 train_time:102692ms step_avg:152.59ms
step:684/1480 train_time:102847ms step_avg:152.59ms
step:685/1480 train_time:103005ms step_avg:152.60ms
step:686/1480 train_time:103163ms step_avg:152.61ms
step:687/1480 train_time:103319ms step_avg:152.61ms
step:688/1480 train_time:103477ms step_avg:152.62ms
step:689/1480 train_time:103636ms step_avg:152.63ms
step:690/1480 train_time:103793ms step_avg:152.64ms
step:691/1480 train_time:103949ms step_avg:152.64ms
step:692/1480 train_time:104106ms step_avg:152.65ms
step:693/1480 train_time:104263ms step_avg:152.65ms
step:694/1480 train_time:104419ms step_avg:152.66ms
step:695/1480 train_time:104576ms step_avg:152.67ms
step:696/1480 train_time:104730ms step_avg:152.67ms
step:697/1480 train_time:104888ms step_avg:152.68ms
step:698/1480 train_time:105043ms step_avg:152.68ms
step:699/1480 train_time:105200ms step_avg:152.69ms
step:700/1480 train_time:105357ms step_avg:152.69ms
step:701/1480 train_time:105513ms step_avg:152.70ms
step:702/1480 train_time:105670ms step_avg:152.70ms
step:703/1480 train_time:105826ms step_avg:152.71ms
step:704/1480 train_time:105983ms step_avg:152.71ms
step:705/1480 train_time:106140ms step_avg:152.72ms
step:706/1480 train_time:106300ms step_avg:152.73ms
step:707/1480 train_time:106457ms step_avg:152.74ms
step:708/1480 train_time:106613ms step_avg:152.74ms
step:709/1480 train_time:106769ms step_avg:152.75ms
step:710/1480 train_time:106926ms step_avg:152.75ms
step:711/1480 train_time:107083ms step_avg:152.76ms
step:712/1480 train_time:107240ms step_avg:152.76ms
step:713/1480 train_time:107397ms step_avg:152.77ms
step:714/1480 train_time:107554ms step_avg:152.78ms
step:715/1480 train_time:107710ms step_avg:152.78ms
step:716/1480 train_time:107865ms step_avg:152.78ms
step:717/1480 train_time:108022ms step_avg:152.79ms
step:718/1480 train_time:108179ms step_avg:152.80ms
step:719/1480 train_time:108335ms step_avg:152.80ms
step:720/1480 train_time:108493ms step_avg:152.81ms
step:721/1480 train_time:108649ms step_avg:152.81ms
step:722/1480 train_time:108807ms step_avg:152.82ms
step:723/1480 train_time:108963ms step_avg:152.82ms
step:724/1480 train_time:109120ms step_avg:152.83ms
step:725/1480 train_time:109277ms step_avg:152.83ms
step:726/1480 train_time:109433ms step_avg:152.84ms
step:727/1480 train_time:109591ms step_avg:152.85ms
step:728/1480 train_time:109746ms step_avg:152.85ms
step:729/1480 train_time:109905ms step_avg:152.86ms
step:730/1480 train_time:110062ms step_avg:152.86ms
step:731/1480 train_time:110219ms step_avg:152.87ms
step:732/1480 train_time:110375ms step_avg:152.87ms
step:733/1480 train_time:110531ms step_avg:152.88ms
step:734/1480 train_time:110689ms step_avg:152.89ms
step:735/1480 train_time:110846ms step_avg:152.89ms
step:736/1480 train_time:111003ms step_avg:152.90ms
step:737/1480 train_time:111159ms step_avg:152.90ms
step:738/1480 train_time:111313ms step_avg:152.90ms
step:739/1480 train_time:111470ms step_avg:152.91ms
step:740/1480 train_time:111629ms step_avg:152.92ms
step:741/1480 train_time:111787ms step_avg:152.92ms
step:742/1480 train_time:111943ms step_avg:152.93ms
step:743/1480 train_time:112099ms step_avg:152.93ms
step:744/1480 train_time:112256ms step_avg:152.94ms
step:745/1480 train_time:112413ms step_avg:152.94ms
step:746/1480 train_time:112569ms step_avg:152.95ms
step:747/1480 train_time:112726ms step_avg:152.95ms
step:748/1480 train_time:112885ms step_avg:152.96ms
step:749/1480 train_time:113041ms step_avg:152.96ms
step:750/1480 train_time:113197ms step_avg:152.97ms
step:750/1480 val_loss:3.5493 train_time:113268ms step_avg:153.06ms
step:751/1480 train_time:113359ms step_avg:152.98ms
step:752/1480 train_time:113515ms step_avg:152.98ms
step:753/1480 train_time:113672ms step_avg:152.99ms
step:754/1480 train_time:113828ms step_avg:153.00ms
step:755/1480 train_time:113984ms step_avg:153.00ms
step:756/1480 train_time:114140ms step_avg:153.00ms
step:757/1480 train_time:114299ms step_avg:153.01ms
step:758/1480 train_time:114456ms step_avg:153.02ms
step:759/1480 train_time:114627ms step_avg:153.04ms
step:760/1480 train_time:114770ms step_avg:153.03ms
step:761/1480 train_time:114926ms step_avg:153.03ms
step:762/1480 train_time:115083ms step_avg:153.04ms
step:763/1480 train_time:115239ms step_avg:153.04ms
step:764/1480 train_time:115396ms step_avg:153.05ms
step:765/1480 train_time:115555ms step_avg:153.05ms
step:766/1480 train_time:115712ms step_avg:153.06ms
step:767/1480 train_time:115869ms step_avg:153.06ms
step:768/1480 train_time:116025ms step_avg:153.07ms
step:769/1480 train_time:116182ms step_avg:153.07ms
step:770/1480 train_time:116338ms step_avg:153.08ms
step:771/1480 train_time:116497ms step_avg:153.08ms
step:772/1480 train_time:116654ms step_avg:153.09ms
step:773/1480 train_time:116811ms step_avg:153.09ms
step:774/1480 train_time:116969ms step_avg:153.10ms
step:775/1480 train_time:117127ms step_avg:153.11ms
step:776/1480 train_time:117285ms step_avg:153.11ms
step:777/1480 train_time:117446ms step_avg:153.12ms
step:778/1480 train_time:117606ms step_avg:153.13ms
step:779/1480 train_time:117765ms step_avg:153.14ms
step:780/1480 train_time:117924ms step_avg:153.15ms
step:781/1480 train_time:118081ms step_avg:153.15ms
step:782/1480 train_time:118238ms step_avg:153.16ms
step:783/1480 train_time:118394ms step_avg:153.16ms
step:784/1480 train_time:118554ms step_avg:153.17ms
step:785/1480 train_time:118711ms step_avg:153.18ms
step:786/1480 train_time:118869ms step_avg:153.18ms
step:787/1480 train_time:119028ms step_avg:153.19ms
step:788/1480 train_time:119187ms step_avg:153.20ms
step:789/1480 train_time:119345ms step_avg:153.20ms
step:790/1480 train_time:119503ms step_avg:153.21ms
step:791/1480 train_time:119663ms step_avg:153.22ms
step:792/1480 train_time:119821ms step_avg:153.22ms
step:793/1480 train_time:119978ms step_avg:153.23ms
step:794/1480 train_time:120136ms step_avg:153.24ms
step:795/1480 train_time:120296ms step_avg:153.24ms
step:796/1480 train_time:120456ms step_avg:153.25ms
step:797/1480 train_time:120614ms step_avg:153.26ms
step:798/1480 train_time:120774ms step_avg:153.27ms
step:799/1480 train_time:120935ms step_avg:153.28ms
step:800/1480 train_time:121093ms step_avg:153.28ms
step:801/1480 train_time:121251ms step_avg:153.29ms
step:802/1480 train_time:121410ms step_avg:153.29ms
step:803/1480 train_time:121568ms step_avg:153.30ms
step:804/1480 train_time:121727ms step_avg:153.31ms
step:805/1480 train_time:121885ms step_avg:153.31ms
step:806/1480 train_time:122043ms step_avg:153.32ms
step:807/1480 train_time:122200ms step_avg:153.33ms
step:808/1480 train_time:122358ms step_avg:153.33ms
step:809/1480 train_time:122514ms step_avg:153.33ms
step:810/1480 train_time:122672ms step_avg:153.34ms
step:811/1480 train_time:122830ms step_avg:153.35ms
step:812/1480 train_time:122988ms step_avg:153.35ms
step:813/1480 train_time:123144ms step_avg:153.36ms
step:814/1480 train_time:123302ms step_avg:153.36ms
step:815/1480 train_time:123459ms step_avg:153.36ms
step:816/1480 train_time:123617ms step_avg:153.37ms
step:817/1480 train_time:123774ms step_avg:153.38ms
step:818/1480 train_time:123931ms step_avg:153.38ms
step:819/1480 train_time:124091ms step_avg:153.39ms
step:820/1480 train_time:124248ms step_avg:153.39ms
step:821/1480 train_time:124406ms step_avg:153.40ms
step:822/1480 train_time:124564ms step_avg:153.40ms
step:823/1480 train_time:124722ms step_avg:153.41ms
step:824/1480 train_time:124879ms step_avg:153.41ms
step:825/1480 train_time:125037ms step_avg:153.42ms
step:826/1480 train_time:125197ms step_avg:153.43ms
step:827/1480 train_time:125355ms step_avg:153.43ms
step:828/1480 train_time:125512ms step_avg:153.44ms
step:829/1480 train_time:125672ms step_avg:153.45ms
step:830/1480 train_time:125831ms step_avg:153.45ms
step:831/1480 train_time:125989ms step_avg:153.46ms
step:832/1480 train_time:126147ms step_avg:153.46ms
step:833/1480 train_time:126305ms step_avg:153.47ms
step:834/1480 train_time:126465ms step_avg:153.48ms
step:835/1480 train_time:126621ms step_avg:153.48ms
step:836/1480 train_time:126779ms step_avg:153.49ms
step:837/1480 train_time:126936ms step_avg:153.49ms
step:838/1480 train_time:127094ms step_avg:153.49ms
step:839/1480 train_time:127251ms step_avg:153.50ms
step:840/1480 train_time:127408ms step_avg:153.50ms
step:841/1480 train_time:127567ms step_avg:153.51ms
step:842/1480 train_time:127726ms step_avg:153.52ms
step:843/1480 train_time:127883ms step_avg:153.52ms
step:844/1480 train_time:128040ms step_avg:153.52ms
step:845/1480 train_time:128197ms step_avg:153.53ms
step:846/1480 train_time:128357ms step_avg:153.54ms
step:847/1480 train_time:128515ms step_avg:153.54ms
step:848/1480 train_time:128673ms step_avg:153.55ms
step:849/1480 train_time:128830ms step_avg:153.55ms
step:850/1480 train_time:128989ms step_avg:153.56ms
step:851/1480 train_time:129148ms step_avg:153.57ms
step:852/1480 train_time:129307ms step_avg:153.57ms
step:853/1480 train_time:129465ms step_avg:153.58ms
step:854/1480 train_time:129624ms step_avg:153.58ms
step:855/1480 train_time:129781ms step_avg:153.59ms
step:856/1480 train_time:129938ms step_avg:153.59ms
step:857/1480 train_time:130097ms step_avg:153.60ms
step:858/1480 train_time:130257ms step_avg:153.60ms
step:859/1480 train_time:130416ms step_avg:153.61ms
step:860/1480 train_time:130574ms step_avg:153.62ms
step:861/1480 train_time:130733ms step_avg:153.62ms
step:862/1480 train_time:130895ms step_avg:153.63ms
step:863/1480 train_time:131055ms step_avg:153.64ms
step:864/1480 train_time:131212ms step_avg:153.64ms
step:865/1480 train_time:131370ms step_avg:153.65ms
step:866/1480 train_time:131530ms step_avg:153.66ms
step:867/1480 train_time:131691ms step_avg:153.66ms
step:868/1480 train_time:131848ms step_avg:153.67ms
step:869/1480 train_time:132006ms step_avg:153.67ms
step:870/1480 train_time:132167ms step_avg:153.68ms
step:871/1480 train_time:132325ms step_avg:153.69ms
step:872/1480 train_time:132483ms step_avg:153.69ms
step:873/1480 train_time:132641ms step_avg:153.70ms
step:874/1480 train_time:132800ms step_avg:153.70ms
step:875/1480 train_time:132959ms step_avg:153.71ms
step:875/1480 val_loss:3.5041 train_time:133031ms step_avg:153.79ms
step:876/1480 train_time:133125ms step_avg:153.72ms
step:877/1480 train_time:133278ms step_avg:153.72ms
step:878/1480 train_time:133435ms step_avg:153.73ms
step:879/1480 train_time:133593ms step_avg:153.73ms
step:880/1480 train_time:133751ms step_avg:153.74ms
step:881/1480 train_time:133909ms step_avg:153.74ms
step:882/1480 train_time:134068ms step_avg:153.75ms
step:883/1480 train_time:134228ms step_avg:153.76ms
step:884/1480 train_time:134390ms step_avg:153.76ms
step:885/1480 train_time:134549ms step_avg:153.77ms
step:886/1480 train_time:134710ms step_avg:153.78ms
step:887/1480 train_time:134869ms step_avg:153.78ms
step:888/1480 train_time:135032ms step_avg:153.80ms
step:889/1480 train_time:135192ms step_avg:153.80ms
step:890/1480 train_time:135349ms step_avg:153.81ms
step:891/1480 train_time:135508ms step_avg:153.81ms
step:892/1480 train_time:135667ms step_avg:153.82ms
step:893/1480 train_time:135826ms step_avg:153.82ms
step:894/1480 train_time:135987ms step_avg:153.83ms
step:895/1480 train_time:136149ms step_avg:153.84ms
step:896/1480 train_time:136308ms step_avg:153.85ms
step:897/1480 train_time:136466ms step_avg:153.85ms
step:898/1480 train_time:136627ms step_avg:153.86ms
step:899/1480 train_time:136786ms step_avg:153.87ms
step:900/1480 train_time:136944ms step_avg:153.87ms
step:901/1480 train_time:137106ms step_avg:153.88ms
step:902/1480 train_time:137265ms step_avg:153.88ms
step:903/1480 train_time:137427ms step_avg:153.89ms
step:904/1480 train_time:137587ms step_avg:153.90ms
step:905/1480 train_time:137745ms step_avg:153.91ms
step:906/1480 train_time:137905ms step_avg:153.91ms
step:907/1480 train_time:138066ms step_avg:153.92ms
step:908/1480 train_time:138225ms step_avg:153.93ms
step:909/1480 train_time:138385ms step_avg:153.93ms
step:910/1480 train_time:138549ms step_avg:153.94ms
step:911/1480 train_time:138708ms step_avg:153.95ms
step:912/1480 train_time:138867ms step_avg:153.95ms
step:913/1480 train_time:139029ms step_avg:153.96ms
step:914/1480 train_time:139189ms step_avg:153.97ms
step:915/1480 train_time:139350ms step_avg:153.98ms
step:916/1480 train_time:139509ms step_avg:153.98ms
step:917/1480 train_time:139667ms step_avg:153.99ms
step:918/1480 train_time:139829ms step_avg:154.00ms
step:919/1480 train_time:139991ms step_avg:154.01ms
step:920/1480 train_time:140149ms step_avg:154.01ms
step:921/1480 train_time:140310ms step_avg:154.02ms
step:922/1480 train_time:140470ms step_avg:154.02ms
step:923/1480 train_time:140629ms step_avg:154.03ms
step:924/1480 train_time:140788ms step_avg:154.04ms
step:925/1480 train_time:140950ms step_avg:154.04ms
step:926/1480 train_time:141111ms step_avg:154.05ms
step:927/1480 train_time:141267ms step_avg:154.05ms
step:928/1480 train_time:141427ms step_avg:154.06ms
step:929/1480 train_time:141587ms step_avg:154.07ms
step:930/1480 train_time:141748ms step_avg:154.07ms
step:931/1480 train_time:141906ms step_avg:154.08ms
step:932/1480 train_time:142064ms step_avg:154.08ms
step:933/1480 train_time:142224ms step_avg:154.09ms
step:934/1480 train_time:142385ms step_avg:154.10ms
step:935/1480 train_time:142546ms step_avg:154.10ms
step:936/1480 train_time:142707ms step_avg:154.11ms
step:937/1480 train_time:142867ms step_avg:154.12ms
step:938/1480 train_time:143027ms step_avg:154.12ms
step:939/1480 train_time:143188ms step_avg:154.13ms
step:940/1480 train_time:143349ms step_avg:154.14ms
step:941/1480 train_time:143508ms step_avg:154.14ms
step:942/1480 train_time:143666ms step_avg:154.15ms
step:943/1480 train_time:143827ms step_avg:154.16ms
step:944/1480 train_time:143990ms step_avg:154.16ms
step:945/1480 train_time:144149ms step_avg:154.17ms
step:946/1480 train_time:144311ms step_avg:154.18ms
step:947/1480 train_time:144471ms step_avg:154.18ms
step:948/1480 train_time:144631ms step_avg:154.19ms
step:949/1480 train_time:144805ms step_avg:154.21ms
step:950/1480 train_time:144950ms step_avg:154.20ms
step:951/1480 train_time:145112ms step_avg:154.21ms
step:952/1480 train_time:145270ms step_avg:154.21ms
step:953/1480 train_time:145431ms step_avg:154.22ms
step:954/1480 train_time:145592ms step_avg:154.23ms
step:955/1480 train_time:145749ms step_avg:154.23ms
step:956/1480 train_time:145908ms step_avg:154.24ms
step:957/1480 train_time:146068ms step_avg:154.24ms
step:958/1480 train_time:146232ms step_avg:154.25ms
step:959/1480 train_time:146390ms step_avg:154.26ms
step:960/1480 train_time:146550ms step_avg:154.26ms
step:961/1480 train_time:146709ms step_avg:154.27ms
step:962/1480 train_time:146867ms step_avg:154.27ms
step:963/1480 train_time:147029ms step_avg:154.28ms
step:964/1480 train_time:147190ms step_avg:154.29ms
step:965/1480 train_time:147349ms step_avg:154.29ms
step:966/1480 train_time:147508ms step_avg:154.30ms
step:967/1480 train_time:147665ms step_avg:154.30ms
step:968/1480 train_time:147825ms step_avg:154.31ms
step:969/1480 train_time:147985ms step_avg:154.31ms
step:970/1480 train_time:148143ms step_avg:154.32ms
step:971/1480 train_time:148303ms step_avg:154.32ms
step:972/1480 train_time:148462ms step_avg:154.33ms
step:973/1480 train_time:148621ms step_avg:154.33ms
step:974/1480 train_time:148782ms step_avg:154.34ms
step:975/1480 train_time:148942ms step_avg:154.34ms
step:976/1480 train_time:149103ms step_avg:154.35ms
step:977/1480 train_time:149263ms step_avg:154.36ms
step:978/1480 train_time:149423ms step_avg:154.36ms
step:979/1480 train_time:149585ms step_avg:154.37ms
step:980/1480 train_time:149746ms step_avg:154.38ms
step:981/1480 train_time:149908ms step_avg:154.39ms
step:982/1480 train_time:150065ms step_avg:154.39ms
step:983/1480 train_time:150229ms step_avg:154.40ms
step:984/1480 train_time:150388ms step_avg:154.40ms
step:985/1480 train_time:150549ms step_avg:154.41ms
step:986/1480 train_time:150708ms step_avg:154.41ms
step:987/1480 train_time:150865ms step_avg:154.42ms
step:988/1480 train_time:151026ms step_avg:154.42ms
step:989/1480 train_time:151187ms step_avg:154.43ms
step:990/1480 train_time:151348ms step_avg:154.44ms
step:991/1480 train_time:151509ms step_avg:154.44ms
step:992/1480 train_time:151673ms step_avg:154.45ms
step:993/1480 train_time:151842ms step_avg:154.47ms
step:994/1480 train_time:152003ms step_avg:154.47ms
step:995/1480 train_time:152163ms step_avg:154.48ms
step:996/1480 train_time:152320ms step_avg:154.48ms
step:997/1480 train_time:152479ms step_avg:154.49ms
step:998/1480 train_time:152638ms step_avg:154.49ms
step:999/1480 train_time:152799ms step_avg:154.50ms
step:1000/1480 train_time:152961ms step_avg:154.51ms
step:1000/1480 val_loss:3.4402 train_time:153034ms step_avg:154.58ms
step:1001/1480 train_time:153127ms step_avg:154.52ms
step:1002/1480 train_time:153286ms step_avg:154.52ms
step:1003/1480 train_time:153448ms step_avg:154.53ms
step:1004/1480 train_time:153610ms step_avg:154.54ms
step:1005/1480 train_time:153771ms step_avg:154.54ms
step:1006/1480 train_time:153931ms step_avg:154.55ms
step:1007/1480 train_time:154091ms step_avg:154.55ms
step:1008/1480 train_time:154251ms step_avg:154.56ms
step:1009/1480 train_time:154417ms step_avg:154.57ms
step:1010/1480 train_time:154576ms step_avg:154.58ms
step:1011/1480 train_time:154736ms step_avg:154.58ms
step:1012/1480 train_time:154894ms step_avg:154.59ms
step:1013/1480 train_time:155056ms step_avg:154.59ms
step:1014/1480 train_time:155215ms step_avg:154.60ms
step:1015/1480 train_time:155377ms step_avg:154.60ms
step:1016/1480 train_time:155536ms step_avg:154.61ms
step:1017/1480 train_time:155697ms step_avg:154.61ms
step:1018/1480 train_time:155858ms step_avg:154.62ms
step:1019/1480 train_time:156019ms step_avg:154.63ms
step:1020/1480 train_time:156179ms step_avg:154.63ms
step:1021/1480 train_time:156339ms step_avg:154.64ms
step:1022/1480 train_time:156499ms step_avg:154.64ms
step:1023/1480 train_time:156661ms step_avg:154.65ms
step:1024/1480 train_time:156822ms step_avg:154.66ms
step:1025/1480 train_time:156984ms step_avg:154.66ms
step:1026/1480 train_time:157145ms step_avg:154.67ms
step:1027/1480 train_time:157305ms step_avg:154.68ms
step:1028/1480 train_time:157467ms step_avg:154.68ms
step:1029/1480 train_time:157630ms step_avg:154.69ms
step:1030/1480 train_time:157790ms step_avg:154.70ms
step:1031/1480 train_time:157949ms step_avg:154.70ms
step:1032/1480 train_time:158111ms step_avg:154.71ms
step:1033/1480 train_time:158270ms step_avg:154.71ms
step:1034/1480 train_time:158431ms step_avg:154.72ms
step:1035/1480 train_time:158592ms step_avg:154.72ms
step:1036/1480 train_time:158752ms step_avg:154.73ms
step:1037/1480 train_time:158910ms step_avg:154.73ms
step:1038/1480 train_time:159071ms step_avg:154.74ms
step:1039/1480 train_time:159234ms step_avg:154.75ms
step:1040/1480 train_time:159392ms step_avg:154.75ms
step:1041/1480 train_time:159551ms step_avg:154.75ms
step:1042/1480 train_time:159709ms step_avg:154.76ms
step:1043/1480 train_time:159869ms step_avg:154.76ms
step:1044/1480 train_time:160028ms step_avg:154.77ms
step:1045/1480 train_time:160191ms step_avg:154.77ms
step:1046/1480 train_time:160351ms step_avg:154.78ms
step:1047/1480 train_time:160509ms step_avg:154.78ms
step:1048/1480 train_time:160670ms step_avg:154.79ms
step:1049/1480 train_time:160830ms step_avg:154.79ms
step:1050/1480 train_time:160991ms step_avg:154.80ms
step:1051/1480 train_time:161153ms step_avg:154.81ms
step:1052/1480 train_time:161312ms step_avg:154.81ms
step:1053/1480 train_time:161472ms step_avg:154.81ms
step:1054/1480 train_time:161632ms step_avg:154.82ms
step:1055/1480 train_time:161791ms step_avg:154.82ms
step:1056/1480 train_time:161951ms step_avg:154.83ms
step:1057/1480 train_time:162111ms step_avg:154.83ms
step:1058/1480 train_time:162272ms step_avg:154.84ms
step:1059/1480 train_time:162435ms step_avg:154.85ms
step:1060/1480 train_time:162598ms step_avg:154.86ms
step:1061/1480 train_time:162757ms step_avg:154.86ms
step:1062/1480 train_time:162916ms step_avg:154.86ms
step:1063/1480 train_time:163075ms step_avg:154.87ms
step:1064/1480 train_time:163233ms step_avg:154.87ms
step:1065/1480 train_time:163393ms step_avg:154.88ms
step:1066/1480 train_time:163555ms step_avg:154.88ms
step:1067/1480 train_time:163720ms step_avg:154.89ms
step:1068/1480 train_time:163881ms step_avg:154.90ms
step:1069/1480 train_time:164046ms step_avg:154.91ms
step:1070/1480 train_time:164205ms step_avg:154.91ms
step:1071/1480 train_time:164369ms step_avg:154.92ms
step:1072/1480 train_time:164529ms step_avg:154.92ms
step:1073/1480 train_time:164688ms step_avg:154.93ms
step:1074/1480 train_time:164848ms step_avg:154.93ms
step:1075/1480 train_time:165009ms step_avg:154.94ms
step:1076/1480 train_time:165169ms step_avg:154.94ms
step:1077/1480 train_time:165329ms step_avg:154.95ms
step:1078/1480 train_time:165496ms step_avg:154.96ms
step:1079/1480 train_time:165659ms step_avg:154.97ms
step:1080/1480 train_time:165820ms step_avg:154.97ms
step:1081/1480 train_time:165982ms step_avg:154.98ms
step:1082/1480 train_time:166143ms step_avg:154.98ms
step:1083/1480 train_time:166303ms step_avg:154.99ms
step:1084/1480 train_time:166464ms step_avg:154.99ms
step:1085/1480 train_time:166626ms step_avg:155.00ms
step:1086/1480 train_time:166786ms step_avg:155.01ms
step:1087/1480 train_time:166948ms step_avg:155.01ms
step:1088/1480 train_time:167108ms step_avg:155.02ms
step:1089/1480 train_time:167272ms step_avg:155.03ms
step:1090/1480 train_time:167435ms step_avg:155.03ms
step:1091/1480 train_time:167594ms step_avg:155.04ms
step:1092/1480 train_time:167756ms step_avg:155.04ms
step:1093/1480 train_time:167915ms step_avg:155.05ms
step:1094/1480 train_time:168075ms step_avg:155.05ms
step:1095/1480 train_time:168234ms step_avg:155.05ms
step:1096/1480 train_time:168397ms step_avg:155.06ms
step:1097/1480 train_time:168557ms step_avg:155.07ms
step:1098/1480 train_time:168721ms step_avg:155.07ms
step:1099/1480 train_time:168882ms step_avg:155.08ms
step:1100/1480 train_time:169047ms step_avg:155.09ms
step:1101/1480 train_time:169209ms step_avg:155.10ms
step:1102/1480 train_time:169372ms step_avg:155.10ms
step:1103/1480 train_time:169537ms step_avg:155.11ms
step:1104/1480 train_time:169700ms step_avg:155.12ms
step:1105/1480 train_time:169862ms step_avg:155.13ms
step:1106/1480 train_time:170026ms step_avg:155.13ms
step:1107/1480 train_time:170188ms step_avg:155.14ms
step:1108/1480 train_time:170348ms step_avg:155.14ms
step:1109/1480 train_time:170508ms step_avg:155.15ms
step:1110/1480 train_time:170669ms step_avg:155.15ms
step:1111/1480 train_time:170832ms step_avg:155.16ms
step:1112/1480 train_time:170993ms step_avg:155.17ms
step:1113/1480 train_time:171162ms step_avg:155.18ms
step:1114/1480 train_time:171325ms step_avg:155.19ms
step:1115/1480 train_time:171487ms step_avg:155.19ms
step:1116/1480 train_time:171648ms step_avg:155.20ms
step:1117/1480 train_time:171811ms step_avg:155.20ms
step:1118/1480 train_time:171977ms step_avg:155.21ms
step:1119/1480 train_time:172138ms step_avg:155.22ms
step:1120/1480 train_time:172299ms step_avg:155.22ms
step:1121/1480 train_time:172461ms step_avg:155.23ms
step:1122/1480 train_time:172623ms step_avg:155.24ms
step:1123/1480 train_time:172784ms step_avg:155.24ms
step:1124/1480 train_time:172948ms step_avg:155.25ms
step:1125/1480 train_time:173108ms step_avg:155.25ms
step:1125/1480 val_loss:3.3842 train_time:173182ms step_avg:155.32ms
step:1126/1480 train_time:173278ms step_avg:155.27ms
step:1127/1480 train_time:173434ms step_avg:155.27ms
step:1128/1480 train_time:173594ms step_avg:155.27ms
step:1129/1480 train_time:173758ms step_avg:155.28ms
step:1130/1480 train_time:173920ms step_avg:155.29ms
step:1131/1480 train_time:174087ms step_avg:155.30ms
step:1132/1480 train_time:174246ms step_avg:155.30ms
step:1133/1480 train_time:174408ms step_avg:155.31ms
step:1134/1480 train_time:174571ms step_avg:155.31ms
step:1135/1480 train_time:174734ms step_avg:155.32ms
step:1136/1480 train_time:174897ms step_avg:155.33ms
step:1137/1480 train_time:175058ms step_avg:155.33ms
step:1138/1480 train_time:175222ms step_avg:155.34ms
step:1139/1480 train_time:175397ms step_avg:155.36ms
step:1140/1480 train_time:175545ms step_avg:155.35ms
step:1141/1480 train_time:175710ms step_avg:155.36ms
step:1142/1480 train_time:175871ms step_avg:155.36ms
step:1143/1480 train_time:176034ms step_avg:155.37ms
step:1144/1480 train_time:176197ms step_avg:155.38ms
step:1145/1480 train_time:176357ms step_avg:155.38ms
step:1146/1480 train_time:176521ms step_avg:155.39ms
step:1147/1480 train_time:176682ms step_avg:155.39ms
step:1148/1480 train_time:176842ms step_avg:155.40ms
step:1149/1480 train_time:177005ms step_avg:155.40ms
step:1150/1480 train_time:177166ms step_avg:155.41ms
step:1151/1480 train_time:177332ms step_avg:155.42ms
step:1152/1480 train_time:177496ms step_avg:155.43ms
step:1153/1480 train_time:177662ms step_avg:155.44ms
step:1154/1480 train_time:177823ms step_avg:155.44ms
step:1155/1480 train_time:177985ms step_avg:155.45ms
step:1156/1480 train_time:178152ms step_avg:155.46ms
step:1157/1480 train_time:178314ms step_avg:155.46ms
step:1158/1480 train_time:178476ms step_avg:155.47ms
step:1159/1480 train_time:178639ms step_avg:155.47ms
step:1160/1480 train_time:178799ms step_avg:155.48ms
step:1161/1480 train_time:178962ms step_avg:155.48ms
step:1162/1480 train_time:179125ms step_avg:155.49ms
step:1163/1480 train_time:179287ms step_avg:155.50ms
step:1164/1480 train_time:179449ms step_avg:155.50ms
step:1165/1480 train_time:179608ms step_avg:155.50ms
step:1166/1480 train_time:179769ms step_avg:155.51ms
step:1167/1480 train_time:179934ms step_avg:155.52ms
step:1168/1480 train_time:180098ms step_avg:155.52ms
step:1169/1480 train_time:180260ms step_avg:155.53ms
step:1170/1480 train_time:180422ms step_avg:155.54ms
step:1171/1480 train_time:180584ms step_avg:155.54ms
step:1172/1480 train_time:180744ms step_avg:155.55ms
step:1173/1480 train_time:180907ms step_avg:155.55ms
step:1174/1480 train_time:181078ms step_avg:155.56ms
step:1175/1480 train_time:181239ms step_avg:155.57ms
step:1176/1480 train_time:181405ms step_avg:155.58ms
step:1177/1480 train_time:181570ms step_avg:155.59ms
step:1178/1480 train_time:181732ms step_avg:155.59ms
step:1179/1480 train_time:181892ms step_avg:155.60ms
step:1180/1480 train_time:182060ms step_avg:155.61ms
step:1181/1480 train_time:182224ms step_avg:155.61ms
step:1182/1480 train_time:182385ms step_avg:155.62ms
step:1183/1480 train_time:182546ms step_avg:155.62ms
step:1184/1480 train_time:182707ms step_avg:155.63ms
step:1185/1480 train_time:182871ms step_avg:155.64ms
step:1186/1480 train_time:183035ms step_avg:155.64ms
step:1187/1480 train_time:183207ms step_avg:155.66ms
step:1188/1480 train_time:183366ms step_avg:155.66ms
step:1189/1480 train_time:183527ms step_avg:155.66ms
step:1190/1480 train_time:183689ms step_avg:155.67ms
step:1191/1480 train_time:183853ms step_avg:155.68ms
step:1192/1480 train_time:184015ms step_avg:155.68ms
step:1193/1480 train_time:184178ms step_avg:155.69ms
step:1194/1480 train_time:184338ms step_avg:155.69ms
step:1195/1480 train_time:184500ms step_avg:155.70ms
step:1196/1480 train_time:184672ms step_avg:155.71ms
step:1197/1480 train_time:184833ms step_avg:155.71ms
step:1198/1480 train_time:185003ms step_avg:155.73ms
step:1199/1480 train_time:185165ms step_avg:155.73ms
step:1200/1480 train_time:185325ms step_avg:155.74ms
step:1201/1480 train_time:185486ms step_avg:155.74ms
step:1202/1480 train_time:185655ms step_avg:155.75ms
step:1203/1480 train_time:185822ms step_avg:155.76ms
step:1204/1480 train_time:185986ms step_avg:155.77ms
step:1205/1480 train_time:186147ms step_avg:155.77ms
step:1206/1480 train_time:186308ms step_avg:155.78ms
step:1207/1480 train_time:186468ms step_avg:155.78ms
step:1208/1480 train_time:186628ms step_avg:155.78ms
step:1209/1480 train_time:186792ms step_avg:155.79ms
step:1210/1480 train_time:186957ms step_avg:155.80ms
step:1211/1480 train_time:187120ms step_avg:155.80ms
step:1212/1480 train_time:187283ms step_avg:155.81ms
step:1213/1480 train_time:187447ms step_avg:155.82ms
step:1214/1480 train_time:187612ms step_avg:155.82ms
step:1215/1480 train_time:187777ms step_avg:155.83ms
step:1216/1480 train_time:187938ms step_avg:155.84ms
step:1217/1480 train_time:188102ms step_avg:155.84ms
step:1218/1480 train_time:188264ms step_avg:155.85ms
step:1219/1480 train_time:188431ms step_avg:155.86ms
step:1220/1480 train_time:188594ms step_avg:155.86ms
step:1221/1480 train_time:188755ms step_avg:155.87ms
step:1222/1480 train_time:188915ms step_avg:155.87ms
step:1223/1480 train_time:189079ms step_avg:155.88ms
step:1224/1480 train_time:189245ms step_avg:155.89ms
step:1225/1480 train_time:189408ms step_avg:155.89ms
step:1226/1480 train_time:189574ms step_avg:155.90ms
step:1227/1480 train_time:189738ms step_avg:155.91ms
step:1228/1480 train_time:189901ms step_avg:155.91ms
step:1229/1480 train_time:190064ms step_avg:155.92ms
step:1230/1480 train_time:190233ms step_avg:155.93ms
step:1231/1480 train_time:190400ms step_avg:155.94ms
step:1232/1480 train_time:190564ms step_avg:155.94ms
step:1233/1480 train_time:190725ms step_avg:155.95ms
step:1234/1480 train_time:190886ms step_avg:155.95ms
step:1235/1480 train_time:191050ms step_avg:155.96ms
step:1236/1480 train_time:191213ms step_avg:155.96ms
step:1237/1480 train_time:191375ms step_avg:155.97ms
step:1238/1480 train_time:191547ms step_avg:155.98ms
step:1239/1480 train_time:191710ms step_avg:155.99ms
step:1240/1480 train_time:191875ms step_avg:156.00ms
step:1241/1480 train_time:192039ms step_avg:156.00ms
step:1242/1480 train_time:192202ms step_avg:156.01ms
step:1243/1480 train_time:192365ms step_avg:156.01ms
step:1244/1480 train_time:192525ms step_avg:156.02ms
step:1245/1480 train_time:192688ms step_avg:156.02ms
step:1246/1480 train_time:192850ms step_avg:156.03ms
step:1247/1480 train_time:193011ms step_avg:156.03ms
step:1248/1480 train_time:193174ms step_avg:156.04ms
step:1249/1480 train_time:193337ms step_avg:156.04ms
step:1250/1480 train_time:193500ms step_avg:156.05ms
step:1250/1480 val_loss:3.3350 train_time:193574ms step_avg:156.11ms
step:1251/1480 train_time:193669ms step_avg:156.06ms
step:1252/1480 train_time:193833ms step_avg:156.06ms
step:1253/1480 train_time:193994ms step_avg:156.07ms
step:1254/1480 train_time:194156ms step_avg:156.07ms
step:1255/1480 train_time:194326ms step_avg:156.09ms
step:1256/1480 train_time:194492ms step_avg:156.09ms
step:1257/1480 train_time:194653ms step_avg:156.10ms
step:1258/1480 train_time:194818ms step_avg:156.10ms
step:1259/1480 train_time:194982ms step_avg:156.11ms
step:1260/1480 train_time:195141ms step_avg:156.11ms
step:1261/1480 train_time:195303ms step_avg:156.12ms
step:1262/1480 train_time:195468ms step_avg:156.12ms
step:1263/1480 train_time:195634ms step_avg:156.13ms
step:1264/1480 train_time:195793ms step_avg:156.13ms
step:1265/1480 train_time:195954ms step_avg:156.14ms
step:1266/1480 train_time:196117ms step_avg:156.14ms
step:1267/1480 train_time:196278ms step_avg:156.15ms
step:1268/1480 train_time:196440ms step_avg:156.15ms
step:1269/1480 train_time:196607ms step_avg:156.16ms
step:1270/1480 train_time:196770ms step_avg:156.17ms
step:1271/1480 train_time:196933ms step_avg:156.17ms
step:1272/1480 train_time:197094ms step_avg:156.18ms
step:1273/1480 train_time:197257ms step_avg:156.18ms
step:1274/1480 train_time:197421ms step_avg:156.19ms
step:1275/1480 train_time:197582ms step_avg:156.19ms
step:1276/1480 train_time:197743ms step_avg:156.19ms
step:1277/1480 train_time:197906ms step_avg:156.20ms
step:1278/1480 train_time:198068ms step_avg:156.20ms
step:1279/1480 train_time:198230ms step_avg:156.21ms
step:1280/1480 train_time:198398ms step_avg:156.22ms
step:1281/1480 train_time:198559ms step_avg:156.22ms
step:1282/1480 train_time:198718ms step_avg:156.22ms
step:1283/1480 train_time:198880ms step_avg:156.23ms
step:1284/1480 train_time:199044ms step_avg:156.24ms
step:1285/1480 train_time:199208ms step_avg:156.24ms
step:1286/1480 train_time:199370ms step_avg:156.25ms
step:1287/1480 train_time:199534ms step_avg:156.25ms
step:1288/1480 train_time:199696ms step_avg:156.26ms
step:1289/1480 train_time:199866ms step_avg:156.27ms
step:1290/1480 train_time:200035ms step_avg:156.28ms
step:1291/1480 train_time:200199ms step_avg:156.28ms
step:1292/1480 train_time:200363ms step_avg:156.29ms
step:1293/1480 train_time:200533ms step_avg:156.30ms
step:1294/1480 train_time:200695ms step_avg:156.30ms
step:1295/1480 train_time:200858ms step_avg:156.31ms
step:1296/1480 train_time:201020ms step_avg:156.31ms
step:1297/1480 train_time:201183ms step_avg:156.32ms
step:1298/1480 train_time:201345ms step_avg:156.32ms
step:1299/1480 train_time:201510ms step_avg:156.33ms
step:1300/1480 train_time:201671ms step_avg:156.33ms
step:1301/1480 train_time:201834ms step_avg:156.34ms
step:1302/1480 train_time:201999ms step_avg:156.35ms
step:1303/1480 train_time:202168ms step_avg:156.36ms
step:1304/1480 train_time:202335ms step_avg:156.36ms
step:1305/1480 train_time:202496ms step_avg:156.37ms
step:1306/1480 train_time:202661ms step_avg:156.37ms
step:1307/1480 train_time:202822ms step_avg:156.38ms
step:1308/1480 train_time:202985ms step_avg:156.38ms
step:1309/1480 train_time:203150ms step_avg:156.39ms
step:1310/1480 train_time:203314ms step_avg:156.40ms
step:1311/1480 train_time:203475ms step_avg:156.40ms
step:1312/1480 train_time:203640ms step_avg:156.41ms
step:1313/1480 train_time:203804ms step_avg:156.41ms
step:1314/1480 train_time:203969ms step_avg:156.42ms
step:1315/1480 train_time:204133ms step_avg:156.42ms
step:1316/1480 train_time:204293ms step_avg:156.43ms
step:1317/1480 train_time:204455ms step_avg:156.43ms
step:1318/1480 train_time:204623ms step_avg:156.44ms
step:1319/1480 train_time:204791ms step_avg:156.45ms
step:1320/1480 train_time:204957ms step_avg:156.46ms
step:1321/1480 train_time:205121ms step_avg:156.46ms
step:1322/1480 train_time:205294ms step_avg:156.47ms
step:1323/1480 train_time:205456ms step_avg:156.48ms
step:1324/1480 train_time:205621ms step_avg:156.48ms
step:1325/1480 train_time:205791ms step_avg:156.50ms
step:1326/1480 train_time:205958ms step_avg:156.50ms
step:1327/1480 train_time:206119ms step_avg:156.51ms
step:1328/1480 train_time:206280ms step_avg:156.51ms
step:1329/1480 train_time:206470ms step_avg:156.54ms
step:1330/1480 train_time:206629ms step_avg:156.54ms
step:1331/1480 train_time:206793ms step_avg:156.54ms
step:1332/1480 train_time:206956ms step_avg:156.55ms
step:1333/1480 train_time:207121ms step_avg:156.55ms
step:1334/1480 train_time:207284ms step_avg:156.56ms
step:1335/1480 train_time:207445ms step_avg:156.56ms
step:1336/1480 train_time:207615ms step_avg:156.57ms
step:1337/1480 train_time:207781ms step_avg:156.58ms
step:1338/1480 train_time:207944ms step_avg:156.58ms
step:1339/1480 train_time:208109ms step_avg:156.59ms
step:1340/1480 train_time:208272ms step_avg:156.60ms
step:1341/1480 train_time:208435ms step_avg:156.60ms
step:1342/1480 train_time:208599ms step_avg:156.61ms
step:1343/1480 train_time:208761ms step_avg:156.61ms
step:1344/1480 train_time:208923ms step_avg:156.61ms
step:1345/1480 train_time:209092ms step_avg:156.62ms
step:1346/1480 train_time:209254ms step_avg:156.63ms
step:1347/1480 train_time:209416ms step_avg:156.63ms
step:1348/1480 train_time:209578ms step_avg:156.63ms
step:1349/1480 train_time:209739ms step_avg:156.64ms
step:1350/1480 train_time:209905ms step_avg:156.65ms
step:1351/1480 train_time:210069ms step_avg:156.65ms
step:1352/1480 train_time:210232ms step_avg:156.66ms
step:1353/1480 train_time:210398ms step_avg:156.66ms
step:1354/1480 train_time:210562ms step_avg:156.67ms
step:1355/1480 train_time:210723ms step_avg:156.67ms
step:1356/1480 train_time:210888ms step_avg:156.68ms
step:1357/1480 train_time:211053ms step_avg:156.68ms
step:1358/1480 train_time:211218ms step_avg:156.69ms
step:1359/1480 train_time:211381ms step_avg:156.69ms
step:1360/1480 train_time:211546ms step_avg:156.70ms
step:1361/1480 train_time:211714ms step_avg:156.71ms
step:1362/1480 train_time:211879ms step_avg:156.71ms
step:1363/1480 train_time:212046ms step_avg:156.72ms
step:1364/1480 train_time:212211ms step_avg:156.73ms
step:1365/1480 train_time:212371ms step_avg:156.73ms
step:1366/1480 train_time:212536ms step_avg:156.74ms
step:1367/1480 train_time:212699ms step_avg:156.74ms
step:1368/1480 train_time:212865ms step_avg:156.75ms
step:1369/1480 train_time:213035ms step_avg:156.76ms
step:1370/1480 train_time:213201ms step_avg:156.77ms
step:1371/1480 train_time:213363ms step_avg:156.77ms
step:1372/1480 train_time:213532ms step_avg:156.78ms
step:1373/1480 train_time:213693ms step_avg:156.78ms
step:1374/1480 train_time:213859ms step_avg:156.79ms
step:1375/1480 train_time:214021ms step_avg:156.79ms
step:1375/1480 val_loss:3.2965 train_time:214096ms step_avg:156.85ms
step:1376/1480 train_time:214188ms step_avg:156.80ms
step:1377/1480 train_time:214353ms step_avg:156.81ms
step:1378/1480 train_time:214516ms step_avg:156.81ms
step:1379/1480 train_time:214682ms step_avg:156.82ms
step:1380/1480 train_time:214845ms step_avg:156.82ms
step:1381/1480 train_time:215012ms step_avg:156.83ms
step:1382/1480 train_time:215177ms step_avg:156.83ms
step:1383/1480 train_time:215340ms step_avg:156.84ms
step:1384/1480 train_time:215506ms step_avg:156.85ms
step:1385/1480 train_time:215665ms step_avg:156.85ms
step:1386/1480 train_time:215826ms step_avg:156.85ms
step:1387/1480 train_time:215991ms step_avg:156.86ms
step:1388/1480 train_time:216152ms step_avg:156.86ms
step:1389/1480 train_time:216319ms step_avg:156.87ms
step:1390/1480 train_time:216480ms step_avg:156.87ms
step:1391/1480 train_time:216642ms step_avg:156.87ms
step:1392/1480 train_time:216805ms step_avg:156.88ms
step:1393/1480 train_time:216967ms step_avg:156.88ms
step:1394/1480 train_time:217133ms step_avg:156.89ms
step:1395/1480 train_time:217297ms step_avg:156.89ms
step:1396/1480 train_time:217461ms step_avg:156.90ms
step:1397/1480 train_time:217622ms step_avg:156.90ms
step:1398/1480 train_time:217782ms step_avg:156.90ms
step:1399/1480 train_time:217945ms step_avg:156.91ms
step:1400/1480 train_time:218113ms step_avg:156.92ms
step:1401/1480 train_time:218273ms step_avg:156.92ms
step:1402/1480 train_time:218435ms step_avg:156.92ms
step:1403/1480 train_time:218601ms step_avg:156.93ms
step:1404/1480 train_time:218763ms step_avg:156.93ms
step:1405/1480 train_time:218930ms step_avg:156.94ms
step:1406/1480 train_time:219095ms step_avg:156.94ms
step:1407/1480 train_time:219257ms step_avg:156.95ms
step:1408/1480 train_time:219419ms step_avg:156.95ms
step:1409/1480 train_time:219591ms step_avg:156.96ms
step:1410/1480 train_time:219755ms step_avg:156.97ms
step:1411/1480 train_time:219915ms step_avg:156.97ms
step:1412/1480 train_time:220077ms step_avg:156.97ms
step:1413/1480 train_time:220241ms step_avg:156.98ms
step:1414/1480 train_time:220404ms step_avg:156.98ms
step:1415/1480 train_time:220569ms step_avg:156.99ms
step:1416/1480 train_time:220743ms step_avg:157.00ms
step:1417/1480 train_time:220908ms step_avg:157.01ms
step:1418/1480 train_time:221073ms step_avg:157.01ms
step:1419/1480 train_time:221240ms step_avg:157.02ms
step:1420/1480 train_time:221404ms step_avg:157.02ms
step:1421/1480 train_time:221568ms step_avg:157.03ms
step:1422/1480 train_time:221734ms step_avg:157.04ms
step:1423/1480 train_time:221896ms step_avg:157.04ms
step:1424/1480 train_time:222063ms step_avg:157.05ms
step:1425/1480 train_time:222234ms step_avg:157.06ms
step:1426/1480 train_time:222399ms step_avg:157.06ms
step:1427/1480 train_time:222565ms step_avg:157.07ms
step:1428/1480 train_time:222727ms step_avg:157.07ms
step:1429/1480 train_time:222887ms step_avg:157.07ms
step:1430/1480 train_time:223052ms step_avg:157.08ms
step:1431/1480 train_time:223219ms step_avg:157.09ms
step:1432/1480 train_time:223386ms step_avg:157.09ms
step:1433/1480 train_time:223556ms step_avg:157.10ms
step:1434/1480 train_time:223725ms step_avg:157.11ms
step:1435/1480 train_time:223891ms step_avg:157.12ms
step:1436/1480 train_time:224057ms step_avg:157.12ms
step:1437/1480 train_time:224219ms step_avg:157.13ms
step:1438/1480 train_time:224379ms step_avg:157.13ms
step:1439/1480 train_time:224545ms step_avg:157.13ms
step:1440/1480 train_time:224708ms step_avg:157.14ms
step:1441/1480 train_time:224873ms step_avg:157.14ms
step:1442/1480 train_time:225040ms step_avg:157.15ms
step:1443/1480 train_time:225212ms step_avg:157.16ms
step:1444/1480 train_time:225377ms step_avg:157.17ms
step:1445/1480 train_time:225540ms step_avg:157.17ms
step:1446/1480 train_time:225707ms step_avg:157.18ms
step:1447/1480 train_time:225875ms step_avg:157.18ms
step:1448/1480 train_time:226038ms step_avg:157.19ms
step:1449/1480 train_time:226202ms step_avg:157.19ms
step:1450/1480 train_time:226365ms step_avg:157.20ms
step:1451/1480 train_time:226528ms step_avg:157.20ms
step:1452/1480 train_time:226693ms step_avg:157.21ms
step:1453/1480 train_time:226858ms step_avg:157.21ms
step:1454/1480 train_time:227020ms step_avg:157.22ms
step:1455/1480 train_time:227190ms step_avg:157.22ms
step:1456/1480 train_time:227354ms step_avg:157.23ms
step:1457/1480 train_time:227517ms step_avg:157.23ms
step:1458/1480 train_time:227680ms step_avg:157.24ms
step:1459/1480 train_time:227846ms step_avg:157.24ms
step:1460/1480 train_time:228008ms step_avg:157.25ms
step:1461/1480 train_time:228172ms step_avg:157.25ms
step:1462/1480 train_time:228335ms step_avg:157.26ms
step:1463/1480 train_time:228500ms step_avg:157.26ms
step:1464/1480 train_time:228667ms step_avg:157.27ms
step:1465/1480 train_time:228833ms step_avg:157.27ms
step:1466/1480 train_time:228996ms step_avg:157.28ms
step:1467/1480 train_time:229163ms step_avg:157.28ms
step:1468/1480 train_time:229325ms step_avg:157.29ms
step:1469/1480 train_time:229488ms step_avg:157.29ms
step:1470/1480 train_time:229657ms step_avg:157.30ms
step:1471/1480 train_time:229829ms step_avg:157.31ms
step:1472/1480 train_time:229998ms step_avg:157.32ms
step:1473/1480 train_time:230161ms step_avg:157.32ms
step:1474/1480 train_time:230328ms step_avg:157.33ms
step:1475/1480 train_time:230497ms step_avg:157.34ms
step:1476/1480 train_time:230660ms step_avg:157.34ms
step:1477/1480 train_time:230828ms step_avg:157.35ms
step:1478/1480 train_time:230998ms step_avg:157.36ms
step:1479/1480 train_time:231163ms step_avg:157.36ms
step:1480/1480 train_time:231326ms step_avg:157.36ms
step:1480/1480 val_loss:3.2777 train_time:231402ms step_avg:157.42ms
peak memory consumption: 34240 MiB