f2546c91-e2b3-4906-9f5b-bee25b47216f.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 08:33:33 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             126W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   28C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   36C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             119W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:28859ms step_avg:nanms
step:2/1480 train_time:29048ms step_avg:nanms
step:3/1480 train_time:29174ms step_avg:nanms
step:4/1480 train_time:29312ms step_avg:nanms
step:5/1480 train_time:29454ms step_avg:nanms
step:6/1480 train_time:29595ms step_avg:nanms
step:7/1480 train_time:29738ms step_avg:nanms
step:8/1480 train_time:29880ms step_avg:nanms
step:9/1480 train_time:30023ms step_avg:nanms
step:10/1480 train_time:30166ms step_avg:nanms
step:11/1480 train_time:143ms step_avg:nanms
step:12/1480 train_time:283ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.80ms
step:14/1480 train_time:569ms step_avg:142.15ms
step:15/1480 train_time:714ms step_avg:142.78ms
step:16/1480 train_time:857ms step_avg:142.89ms
step:17/1480 train_time:999ms step_avg:142.73ms
step:18/1480 train_time:1141ms step_avg:142.65ms
step:19/1480 train_time:1283ms step_avg:142.57ms
step:20/1480 train_time:1427ms step_avg:142.65ms
step:21/1480 train_time:1569ms step_avg:142.65ms
step:22/1480 train_time:1713ms step_avg:142.77ms
step:23/1480 train_time:1856ms step_avg:142.76ms
step:24/1480 train_time:1998ms step_avg:142.70ms
step:25/1480 train_time:2140ms step_avg:142.64ms
step:26/1480 train_time:2282ms step_avg:142.64ms
step:27/1480 train_time:2425ms step_avg:142.67ms
step:28/1480 train_time:2567ms step_avg:142.63ms
step:29/1480 train_time:2714ms step_avg:142.85ms
step:30/1480 train_time:3229ms step_avg:161.45ms
step:31/1480 train_time:3330ms step_avg:158.58ms
step:32/1480 train_time:3473ms step_avg:157.88ms
step:33/1480 train_time:3616ms step_avg:157.21ms
step:34/1480 train_time:3759ms step_avg:156.61ms
step:35/1480 train_time:3900ms step_avg:155.98ms
step:36/1480 train_time:4042ms step_avg:155.44ms
step:37/1480 train_time:4187ms step_avg:155.07ms
step:38/1480 train_time:4333ms step_avg:154.74ms
step:39/1480 train_time:4477ms step_avg:154.38ms
step:40/1480 train_time:4619ms step_avg:153.96ms
step:41/1480 train_time:4760ms step_avg:153.56ms
step:42/1480 train_time:4902ms step_avg:153.18ms
step:43/1480 train_time:5043ms step_avg:152.82ms
step:44/1480 train_time:5186ms step_avg:152.52ms
step:45/1480 train_time:5329ms step_avg:152.26ms
step:46/1480 train_time:5472ms step_avg:152.01ms
step:47/1480 train_time:5615ms step_avg:151.77ms
step:48/1480 train_time:5758ms step_avg:151.52ms
step:49/1480 train_time:5900ms step_avg:151.29ms
step:50/1480 train_time:6042ms step_avg:151.06ms
step:51/1480 train_time:6183ms step_avg:150.81ms
step:52/1480 train_time:6328ms step_avg:150.66ms
step:53/1480 train_time:6471ms step_avg:150.48ms
step:54/1480 train_time:6615ms step_avg:150.33ms
step:55/1480 train_time:6757ms step_avg:150.15ms
step:56/1480 train_time:6900ms step_avg:150.00ms
step:57/1480 train_time:7045ms step_avg:149.89ms
step:58/1480 train_time:7187ms step_avg:149.72ms
step:59/1480 train_time:7330ms step_avg:149.59ms
step:60/1480 train_time:7476ms step_avg:149.53ms
step:61/1480 train_time:7618ms step_avg:149.37ms
step:62/1480 train_time:7759ms step_avg:149.22ms
step:63/1480 train_time:7901ms step_avg:149.07ms
step:64/1480 train_time:8043ms step_avg:148.94ms
step:65/1480 train_time:8185ms step_avg:148.82ms
step:66/1480 train_time:8329ms step_avg:148.73ms
step:67/1480 train_time:8473ms step_avg:148.65ms
step:68/1480 train_time:8616ms step_avg:148.56ms
step:69/1480 train_time:8760ms step_avg:148.47ms
step:70/1480 train_time:8901ms step_avg:148.35ms
step:71/1480 train_time:9043ms step_avg:148.25ms
step:72/1480 train_time:9185ms step_avg:148.15ms
step:73/1480 train_time:9329ms step_avg:148.09ms
step:74/1480 train_time:9472ms step_avg:148.01ms
step:75/1480 train_time:9615ms step_avg:147.93ms
step:76/1480 train_time:9757ms step_avg:147.84ms
step:77/1480 train_time:9900ms step_avg:147.76ms
step:78/1480 train_time:10042ms step_avg:147.67ms
step:79/1480 train_time:10570ms step_avg:153.19ms
step:80/1480 train_time:11074ms step_avg:158.20ms
step:81/1480 train_time:11569ms step_avg:162.94ms
step:82/1480 train_time:11667ms step_avg:162.04ms
step:83/1480 train_time:11810ms step_avg:161.78ms
step:84/1480 train_time:11951ms step_avg:161.51ms
step:85/1480 train_time:12093ms step_avg:161.24ms
step:86/1480 train_time:12236ms step_avg:161.00ms
step:87/1480 train_time:12378ms step_avg:160.76ms
step:88/1480 train_time:12521ms step_avg:160.53ms
step:89/1480 train_time:12666ms step_avg:160.32ms
step:90/1480 train_time:12809ms step_avg:160.11ms
step:91/1480 train_time:12952ms step_avg:159.91ms
step:92/1480 train_time:13096ms step_avg:159.71ms
step:93/1480 train_time:13239ms step_avg:159.51ms
step:94/1480 train_time:13381ms step_avg:159.30ms
step:95/1480 train_time:13524ms step_avg:159.10ms
step:96/1480 train_time:13667ms step_avg:158.91ms
step:97/1480 train_time:13809ms step_avg:158.72ms
step:98/1480 train_time:13951ms step_avg:158.54ms
step:99/1480 train_time:14094ms step_avg:158.36ms
step:100/1480 train_time:14238ms step_avg:158.21ms
step:101/1480 train_time:14386ms step_avg:158.09ms
step:102/1480 train_time:14523ms step_avg:157.86ms
step:103/1480 train_time:14665ms step_avg:157.69ms
step:104/1480 train_time:14809ms step_avg:157.54ms
step:105/1480 train_time:14954ms step_avg:157.41ms
step:106/1480 train_time:15095ms step_avg:157.24ms
step:107/1480 train_time:15239ms step_avg:157.11ms
step:108/1480 train_time:15384ms step_avg:156.97ms
step:109/1480 train_time:15525ms step_avg:156.81ms
step:110/1480 train_time:15668ms step_avg:156.68ms
step:111/1480 train_time:15812ms step_avg:156.56ms
step:112/1480 train_time:15958ms step_avg:156.45ms
step:113/1480 train_time:16103ms step_avg:156.34ms
step:114/1480 train_time:16250ms step_avg:156.25ms
step:115/1480 train_time:16396ms step_avg:156.15ms
step:116/1480 train_time:16541ms step_avg:156.04ms
step:117/1480 train_time:16687ms step_avg:155.95ms
step:118/1480 train_time:16833ms step_avg:155.86ms
step:119/1480 train_time:16979ms step_avg:155.77ms
step:120/1480 train_time:17124ms step_avg:155.67ms
step:121/1480 train_time:17269ms step_avg:155.58ms
step:122/1480 train_time:17416ms step_avg:155.50ms
step:123/1480 train_time:17561ms step_avg:155.41ms
step:124/1480 train_time:17706ms step_avg:155.32ms
step:125/1480 train_time:17853ms step_avg:155.24ms
step:125/1480 val_loss:4.4154 train_time:17918ms step_avg:155.81ms
step:126/1480 train_time:18011ms step_avg:155.27ms
step:127/1480 train_time:18156ms step_avg:155.18ms
step:128/1480 train_time:18301ms step_avg:155.10ms
step:129/1480 train_time:18446ms step_avg:155.01ms
step:130/1480 train_time:18592ms step_avg:154.93ms
step:131/1480 train_time:18738ms step_avg:154.86ms
step:132/1480 train_time:18882ms step_avg:154.77ms
step:133/1480 train_time:19029ms step_avg:154.70ms
step:134/1480 train_time:19176ms step_avg:154.65ms
step:135/1480 train_time:19321ms step_avg:154.57ms
step:136/1480 train_time:19467ms step_avg:154.50ms
step:137/1480 train_time:19614ms step_avg:154.44ms
step:138/1480 train_time:19759ms step_avg:154.37ms
step:139/1480 train_time:19904ms step_avg:154.29ms
step:140/1480 train_time:20050ms step_avg:154.23ms
step:141/1480 train_time:20197ms step_avg:154.18ms
step:142/1480 train_time:20342ms step_avg:154.11ms
step:143/1480 train_time:20488ms step_avg:154.04ms
step:144/1480 train_time:20634ms step_avg:153.99ms
step:145/1480 train_time:20780ms step_avg:153.93ms
step:146/1480 train_time:20925ms step_avg:153.86ms
step:147/1480 train_time:21072ms step_avg:153.81ms
step:148/1480 train_time:21219ms step_avg:153.76ms
step:149/1480 train_time:21364ms step_avg:153.70ms
step:150/1480 train_time:21509ms step_avg:153.63ms
step:151/1480 train_time:21655ms step_avg:153.58ms
step:152/1480 train_time:21801ms step_avg:153.52ms
step:153/1480 train_time:21945ms step_avg:153.46ms
step:154/1480 train_time:22093ms step_avg:153.43ms
step:155/1480 train_time:22239ms step_avg:153.37ms
step:156/1480 train_time:22385ms step_avg:153.32ms
step:157/1480 train_time:22531ms step_avg:153.27ms
step:158/1480 train_time:22677ms step_avg:153.22ms
step:159/1480 train_time:22822ms step_avg:153.17ms
step:160/1480 train_time:22968ms step_avg:153.12ms
step:161/1480 train_time:23116ms step_avg:153.09ms
step:162/1480 train_time:23261ms step_avg:153.03ms
step:163/1480 train_time:23407ms step_avg:152.99ms
step:164/1480 train_time:23554ms step_avg:152.95ms
step:165/1480 train_time:23699ms step_avg:152.90ms
step:166/1480 train_time:23844ms step_avg:152.85ms
step:167/1480 train_time:23991ms step_avg:152.81ms
step:168/1480 train_time:24137ms step_avg:152.77ms
step:169/1480 train_time:24283ms step_avg:152.72ms
step:170/1480 train_time:24428ms step_avg:152.67ms
step:171/1480 train_time:24574ms step_avg:152.64ms
step:172/1480 train_time:24720ms step_avg:152.59ms
step:173/1480 train_time:24866ms step_avg:152.55ms
step:174/1480 train_time:25011ms step_avg:152.51ms
step:175/1480 train_time:25158ms step_avg:152.47ms
step:176/1480 train_time:25303ms step_avg:152.43ms
step:177/1480 train_time:25448ms step_avg:152.39ms
step:178/1480 train_time:25594ms step_avg:152.35ms
step:179/1480 train_time:25739ms step_avg:152.30ms
step:180/1480 train_time:25887ms step_avg:152.28ms
step:181/1480 train_time:26034ms step_avg:152.25ms
step:182/1480 train_time:26180ms step_avg:152.21ms
step:183/1480 train_time:26324ms step_avg:152.16ms
step:184/1480 train_time:26470ms step_avg:152.13ms
step:185/1480 train_time:26617ms step_avg:152.10ms
step:186/1480 train_time:26762ms step_avg:152.06ms
step:187/1480 train_time:26908ms step_avg:152.02ms
step:188/1480 train_time:27054ms step_avg:151.99ms
step:189/1480 train_time:27223ms step_avg:152.09ms
step:190/1480 train_time:27345ms step_avg:151.92ms
step:191/1480 train_time:27492ms step_avg:151.89ms
step:192/1480 train_time:27637ms step_avg:151.85ms
step:193/1480 train_time:27782ms step_avg:151.81ms
step:194/1480 train_time:27928ms step_avg:151.78ms
step:195/1480 train_time:28077ms step_avg:151.77ms
step:196/1480 train_time:28222ms step_avg:151.73ms
step:197/1480 train_time:28368ms step_avg:151.70ms
step:198/1480 train_time:28515ms step_avg:151.67ms
step:199/1480 train_time:28660ms step_avg:151.64ms
step:200/1480 train_time:28804ms step_avg:151.60ms
step:201/1480 train_time:28950ms step_avg:151.57ms
step:202/1480 train_time:29098ms step_avg:151.55ms
step:203/1480 train_time:29243ms step_avg:151.52ms
step:204/1480 train_time:29388ms step_avg:151.48ms
step:205/1480 train_time:29534ms step_avg:151.46ms
step:206/1480 train_time:29680ms step_avg:151.43ms
step:207/1480 train_time:29825ms step_avg:151.39ms
step:208/1480 train_time:29971ms step_avg:151.37ms
step:209/1480 train_time:30118ms step_avg:151.35ms
step:210/1480 train_time:30264ms step_avg:151.32ms
step:211/1480 train_time:30410ms step_avg:151.29ms
step:212/1480 train_time:30557ms step_avg:151.27ms
step:213/1480 train_time:30702ms step_avg:151.24ms
step:214/1480 train_time:30848ms step_avg:151.21ms
step:215/1480 train_time:30994ms step_avg:151.19ms
step:216/1480 train_time:31140ms step_avg:151.17ms
step:217/1480 train_time:31287ms step_avg:151.14ms
step:218/1480 train_time:31433ms step_avg:151.12ms
step:219/1480 train_time:31579ms step_avg:151.10ms
step:220/1480 train_time:31725ms step_avg:151.07ms
step:221/1480 train_time:32277ms step_avg:152.97ms
step:222/1480 train_time:32384ms step_avg:152.75ms
step:223/1480 train_time:32532ms step_avg:152.73ms
step:224/1480 train_time:32681ms step_avg:152.71ms
step:225/1480 train_time:32829ms step_avg:152.69ms
step:226/1480 train_time:32978ms step_avg:152.68ms
step:227/1480 train_time:33125ms step_avg:152.65ms
step:228/1480 train_time:33276ms step_avg:152.64ms
step:229/1480 train_time:33424ms step_avg:152.62ms
step:230/1480 train_time:33572ms step_avg:152.60ms
step:231/1480 train_time:33721ms step_avg:152.58ms
step:232/1480 train_time:33869ms step_avg:152.56ms
step:233/1480 train_time:34018ms step_avg:152.55ms
step:234/1480 train_time:34165ms step_avg:152.52ms
step:235/1480 train_time:34314ms step_avg:152.51ms
step:236/1480 train_time:34462ms step_avg:152.49ms
step:237/1480 train_time:34610ms step_avg:152.47ms
step:238/1480 train_time:34759ms step_avg:152.45ms
step:239/1480 train_time:34908ms step_avg:152.44ms
step:240/1480 train_time:35058ms step_avg:152.43ms
step:241/1480 train_time:35206ms step_avg:152.41ms
step:242/1480 train_time:35355ms step_avg:152.39ms
step:243/1480 train_time:35503ms step_avg:152.37ms
step:244/1480 train_time:35651ms step_avg:152.35ms
step:245/1480 train_time:35800ms step_avg:152.34ms
step:246/1480 train_time:35949ms step_avg:152.32ms
step:247/1480 train_time:36097ms step_avg:152.31ms
step:248/1480 train_time:36246ms step_avg:152.29ms
step:249/1480 train_time:36395ms step_avg:152.28ms
step:250/1480 train_time:36542ms step_avg:152.26ms
step:250/1480 val_loss:4.0014 train_time:36608ms step_avg:152.53ms
step:251/1480 train_time:36701ms step_avg:152.29ms
step:252/1480 train_time:36848ms step_avg:152.26ms
step:253/1480 train_time:36997ms step_avg:152.25ms
step:254/1480 train_time:37146ms step_avg:152.24ms
step:255/1480 train_time:37293ms step_avg:152.22ms
step:256/1480 train_time:37442ms step_avg:152.20ms
step:257/1480 train_time:37589ms step_avg:152.18ms
step:258/1480 train_time:37740ms step_avg:152.18ms
step:259/1480 train_time:37888ms step_avg:152.16ms
step:260/1480 train_time:38036ms step_avg:152.14ms
step:261/1480 train_time:38185ms step_avg:152.13ms
step:262/1480 train_time:38333ms step_avg:152.12ms
step:263/1480 train_time:38482ms step_avg:152.10ms
step:264/1480 train_time:38630ms step_avg:152.09ms
step:265/1480 train_time:38779ms step_avg:152.08ms
step:266/1480 train_time:38927ms step_avg:152.06ms
step:267/1480 train_time:39076ms step_avg:152.05ms
step:268/1480 train_time:39225ms step_avg:152.03ms
step:269/1480 train_time:39373ms step_avg:152.02ms
step:270/1480 train_time:39521ms step_avg:152.00ms
step:271/1480 train_time:39669ms step_avg:151.99ms
step:272/1480 train_time:39818ms step_avg:151.98ms
step:273/1480 train_time:39967ms step_avg:151.96ms
step:274/1480 train_time:40116ms step_avg:151.96ms
step:275/1480 train_time:40264ms step_avg:151.94ms
step:276/1480 train_time:40413ms step_avg:151.93ms
step:277/1480 train_time:40561ms step_avg:151.91ms
step:278/1480 train_time:40709ms step_avg:151.90ms
step:279/1480 train_time:40857ms step_avg:151.89ms
step:280/1480 train_time:41006ms step_avg:151.87ms
step:281/1480 train_time:41155ms step_avg:151.86ms
step:282/1480 train_time:41304ms step_avg:151.85ms
step:283/1480 train_time:41452ms step_avg:151.84ms
step:284/1480 train_time:41601ms step_avg:151.83ms
step:285/1480 train_time:41748ms step_avg:151.81ms
step:286/1480 train_time:41897ms step_avg:151.80ms
step:287/1480 train_time:42046ms step_avg:151.79ms
step:288/1480 train_time:42195ms step_avg:151.78ms
step:289/1480 train_time:42344ms step_avg:151.77ms
step:290/1480 train_time:42492ms step_avg:151.76ms
step:291/1480 train_time:42641ms step_avg:151.75ms
step:292/1480 train_time:42788ms step_avg:151.73ms
step:293/1480 train_time:42937ms step_avg:151.72ms
step:294/1480 train_time:43086ms step_avg:151.71ms
step:295/1480 train_time:43234ms step_avg:151.70ms
step:296/1480 train_time:43383ms step_avg:151.69ms
step:297/1480 train_time:43531ms step_avg:151.68ms
step:298/1480 train_time:43680ms step_avg:151.67ms
step:299/1480 train_time:43827ms step_avg:151.65ms
step:300/1480 train_time:43977ms step_avg:151.64ms
step:301/1480 train_time:44126ms step_avg:151.63ms
step:302/1480 train_time:44274ms step_avg:151.62ms
step:303/1480 train_time:44424ms step_avg:151.62ms
step:304/1480 train_time:44573ms step_avg:151.61ms
step:305/1480 train_time:44722ms step_avg:151.60ms
step:306/1480 train_time:44870ms step_avg:151.59ms
step:307/1480 train_time:45020ms step_avg:151.58ms
step:308/1480 train_time:45168ms step_avg:151.57ms
step:309/1480 train_time:45317ms step_avg:151.56ms
step:310/1480 train_time:45465ms step_avg:151.55ms
step:311/1480 train_time:45614ms step_avg:151.54ms
step:312/1480 train_time:45761ms step_avg:151.53ms
step:313/1480 train_time:45911ms step_avg:151.52ms
step:314/1480 train_time:46060ms step_avg:151.51ms
step:315/1480 train_time:46208ms step_avg:151.50ms
step:316/1480 train_time:46357ms step_avg:151.49ms
step:317/1480 train_time:46506ms step_avg:151.48ms
step:318/1480 train_time:46653ms step_avg:151.47ms
step:319/1480 train_time:46802ms step_avg:151.46ms
step:320/1480 train_time:46950ms step_avg:151.45ms
step:321/1480 train_time:47099ms step_avg:151.44ms
step:322/1480 train_time:47246ms step_avg:151.43ms
step:323/1480 train_time:47396ms step_avg:151.42ms
step:324/1480 train_time:47545ms step_avg:151.42ms
step:325/1480 train_time:47693ms step_avg:151.41ms
step:326/1480 train_time:47843ms step_avg:151.40ms
step:327/1480 train_time:47991ms step_avg:151.39ms
step:328/1480 train_time:48140ms step_avg:151.38ms
step:329/1480 train_time:48288ms step_avg:151.37ms
step:330/1480 train_time:48438ms step_avg:151.37ms
step:331/1480 train_time:48588ms step_avg:151.36ms
step:332/1480 train_time:48740ms step_avg:151.37ms
step:333/1480 train_time:48889ms step_avg:151.36ms
step:334/1480 train_time:49041ms step_avg:151.36ms
step:335/1480 train_time:49190ms step_avg:151.36ms
step:336/1480 train_time:49342ms step_avg:151.36ms
step:337/1480 train_time:49492ms step_avg:151.35ms
step:338/1480 train_time:49643ms step_avg:151.35ms
step:339/1480 train_time:49793ms step_avg:151.35ms
step:340/1480 train_time:49945ms step_avg:151.35ms
step:341/1480 train_time:50096ms step_avg:151.35ms
step:342/1480 train_time:50246ms step_avg:151.34ms
step:343/1480 train_time:50397ms step_avg:151.34ms
step:344/1480 train_time:50548ms step_avg:151.34ms
step:345/1480 train_time:50700ms step_avg:151.34ms
step:346/1480 train_time:50850ms step_avg:151.34ms
step:347/1480 train_time:51002ms step_avg:151.34ms
step:348/1480 train_time:51153ms step_avg:151.34ms
step:349/1480 train_time:51304ms step_avg:151.34ms
step:350/1480 train_time:51454ms step_avg:151.34ms
step:351/1480 train_time:51605ms step_avg:151.34ms
step:352/1480 train_time:51756ms step_avg:151.33ms
step:353/1480 train_time:51907ms step_avg:151.33ms
step:354/1480 train_time:52058ms step_avg:151.33ms
step:355/1480 train_time:52208ms step_avg:151.33ms
step:356/1480 train_time:52360ms step_avg:151.33ms
step:357/1480 train_time:52509ms step_avg:151.32ms
step:358/1480 train_time:52660ms step_avg:151.32ms
step:359/1480 train_time:52812ms step_avg:151.32ms
step:360/1480 train_time:52964ms step_avg:151.33ms
step:361/1480 train_time:53116ms step_avg:151.33ms
step:362/1480 train_time:53267ms step_avg:151.33ms
step:363/1480 train_time:53417ms step_avg:151.32ms
step:364/1480 train_time:53568ms step_avg:151.32ms
step:365/1480 train_time:53719ms step_avg:151.32ms
step:366/1480 train_time:53870ms step_avg:151.32ms
step:367/1480 train_time:54021ms step_avg:151.32ms
step:368/1480 train_time:54171ms step_avg:151.31ms
step:369/1480 train_time:54322ms step_avg:151.32ms
step:370/1480 train_time:54472ms step_avg:151.31ms
step:371/1480 train_time:54624ms step_avg:151.31ms
step:372/1480 train_time:54774ms step_avg:151.31ms
step:373/1480 train_time:54925ms step_avg:151.31ms
step:374/1480 train_time:55077ms step_avg:151.31ms
step:375/1480 train_time:55227ms step_avg:151.31ms
step:375/1480 val_loss:3.8151 train_time:55295ms step_avg:151.49ms
step:376/1480 train_time:55386ms step_avg:151.33ms
step:377/1480 train_time:55535ms step_avg:151.32ms
step:378/1480 train_time:55687ms step_avg:151.32ms
step:379/1480 train_time:55855ms step_avg:151.37ms
step:380/1480 train_time:55988ms step_avg:151.32ms
step:381/1480 train_time:56137ms step_avg:151.31ms
step:382/1480 train_time:56289ms step_avg:151.32ms
step:383/1480 train_time:56440ms step_avg:151.31ms
step:384/1480 train_time:56592ms step_avg:151.32ms
step:385/1480 train_time:56742ms step_avg:151.31ms
step:386/1480 train_time:56893ms step_avg:151.31ms
step:387/1480 train_time:57044ms step_avg:151.31ms
step:388/1480 train_time:57194ms step_avg:151.31ms
step:389/1480 train_time:57346ms step_avg:151.31ms
step:390/1480 train_time:57497ms step_avg:151.31ms
step:391/1480 train_time:57648ms step_avg:151.31ms
step:392/1480 train_time:57798ms step_avg:151.30ms
step:393/1480 train_time:57950ms step_avg:151.31ms
step:394/1480 train_time:58101ms step_avg:151.30ms
step:395/1480 train_time:58252ms step_avg:151.30ms
step:396/1480 train_time:58403ms step_avg:151.30ms
step:397/1480 train_time:58554ms step_avg:151.30ms
step:398/1480 train_time:58705ms step_avg:151.30ms
step:399/1480 train_time:58856ms step_avg:151.30ms
step:400/1480 train_time:59008ms step_avg:151.30ms
step:401/1480 train_time:59158ms step_avg:151.30ms
step:402/1480 train_time:59309ms step_avg:151.30ms
step:403/1480 train_time:59459ms step_avg:151.29ms
step:404/1480 train_time:59610ms step_avg:151.30ms
step:405/1480 train_time:59761ms step_avg:151.29ms
step:406/1480 train_time:59912ms step_avg:151.29ms
step:407/1480 train_time:60063ms step_avg:151.29ms
step:408/1480 train_time:60214ms step_avg:151.29ms
step:409/1480 train_time:60366ms step_avg:151.29ms
step:410/1480 train_time:60516ms step_avg:151.29ms
step:411/1480 train_time:60668ms step_avg:151.29ms
step:412/1480 train_time:60818ms step_avg:151.29ms
step:413/1480 train_time:60969ms step_avg:151.29ms
step:414/1480 train_time:61120ms step_avg:151.29ms
step:415/1480 train_time:61271ms step_avg:151.29ms
step:416/1480 train_time:61423ms step_avg:151.29ms
step:417/1480 train_time:61574ms step_avg:151.29ms
step:418/1480 train_time:61725ms step_avg:151.29ms
step:419/1480 train_time:61875ms step_avg:151.28ms
step:420/1480 train_time:62027ms step_avg:151.28ms
step:421/1480 train_time:62176ms step_avg:151.28ms
step:422/1480 train_time:62329ms step_avg:151.28ms
step:423/1480 train_time:62479ms step_avg:151.28ms
step:424/1480 train_time:62630ms step_avg:151.28ms
step:425/1480 train_time:62780ms step_avg:151.28ms
step:426/1480 train_time:62932ms step_avg:151.28ms
step:427/1480 train_time:63083ms step_avg:151.28ms
step:428/1480 train_time:63234ms step_avg:151.28ms
step:429/1480 train_time:63386ms step_avg:151.28ms
step:430/1480 train_time:63536ms step_avg:151.28ms
step:431/1480 train_time:63687ms step_avg:151.28ms
step:432/1480 train_time:63838ms step_avg:151.27ms
step:433/1480 train_time:63988ms step_avg:151.27ms
step:434/1480 train_time:64138ms step_avg:151.27ms
step:435/1480 train_time:64289ms step_avg:151.27ms
step:436/1480 train_time:64439ms step_avg:151.27ms
step:437/1480 train_time:64590ms step_avg:151.27ms
step:438/1480 train_time:64740ms step_avg:151.26ms
step:439/1480 train_time:64891ms step_avg:151.26ms
step:440/1480 train_time:65042ms step_avg:151.26ms
step:441/1480 train_time:65195ms step_avg:151.26ms
step:442/1480 train_time:65348ms step_avg:151.27ms
step:443/1480 train_time:65501ms step_avg:151.27ms
step:444/1480 train_time:65653ms step_avg:151.27ms
step:445/1480 train_time:65806ms step_avg:151.28ms
step:446/1480 train_time:65958ms step_avg:151.28ms
step:447/1480 train_time:66111ms step_avg:151.28ms
step:448/1480 train_time:66263ms step_avg:151.29ms
step:449/1480 train_time:66415ms step_avg:151.29ms
step:450/1480 train_time:66568ms step_avg:151.29ms
step:451/1480 train_time:66722ms step_avg:151.30ms
step:452/1480 train_time:66874ms step_avg:151.30ms
step:453/1480 train_time:67028ms step_avg:151.30ms
step:454/1480 train_time:67180ms step_avg:151.31ms
step:455/1480 train_time:67334ms step_avg:151.31ms
step:456/1480 train_time:67488ms step_avg:151.32ms
step:457/1480 train_time:67640ms step_avg:151.32ms
step:458/1480 train_time:67793ms step_avg:151.32ms
step:459/1480 train_time:67946ms step_avg:151.33ms
step:460/1480 train_time:68097ms step_avg:151.33ms
step:461/1480 train_time:68251ms step_avg:151.33ms
step:462/1480 train_time:68404ms step_avg:151.34ms
step:463/1480 train_time:68557ms step_avg:151.34ms
step:464/1480 train_time:68710ms step_avg:151.34ms
step:465/1480 train_time:68862ms step_avg:151.34ms
step:466/1480 train_time:69015ms step_avg:151.35ms
step:467/1480 train_time:69168ms step_avg:151.35ms
step:468/1480 train_time:69321ms step_avg:151.35ms
step:469/1480 train_time:69472ms step_avg:151.36ms
step:470/1480 train_time:69626ms step_avg:151.36ms
step:471/1480 train_time:69779ms step_avg:151.36ms
step:472/1480 train_time:69932ms step_avg:151.37ms
step:473/1480 train_time:70085ms step_avg:151.37ms
step:474/1480 train_time:70237ms step_avg:151.37ms
step:475/1480 train_time:70391ms step_avg:151.38ms
step:476/1480 train_time:70542ms step_avg:151.38ms
step:477/1480 train_time:70695ms step_avg:151.38ms
step:478/1480 train_time:70848ms step_avg:151.39ms
step:479/1480 train_time:71002ms step_avg:151.39ms
step:480/1480 train_time:71156ms step_avg:151.39ms
step:481/1480 train_time:71309ms step_avg:151.40ms
step:482/1480 train_time:71460ms step_avg:151.40ms
step:483/1480 train_time:71613ms step_avg:151.40ms
step:484/1480 train_time:71766ms step_avg:151.40ms
step:485/1480 train_time:71918ms step_avg:151.41ms
step:486/1480 train_time:72072ms step_avg:151.41ms
step:487/1480 train_time:72225ms step_avg:151.42ms
step:488/1480 train_time:72378ms step_avg:151.42ms
step:489/1480 train_time:72532ms step_avg:151.42ms
step:490/1480 train_time:72684ms step_avg:151.42ms
step:491/1480 train_time:72836ms step_avg:151.43ms
step:492/1480 train_time:72990ms step_avg:151.43ms
step:493/1480 train_time:73142ms step_avg:151.43ms
step:494/1480 train_time:73295ms step_avg:151.44ms
step:495/1480 train_time:73449ms step_avg:151.44ms
step:496/1480 train_time:73601ms step_avg:151.44ms
step:497/1480 train_time:73755ms step_avg:151.45ms
step:498/1480 train_time:73907ms step_avg:151.45ms
step:499/1480 train_time:74060ms step_avg:151.45ms
step:500/1480 train_time:74213ms step_avg:151.46ms
step:500/1480 val_loss:3.6950 train_time:74283ms step_avg:151.60ms
step:501/1480 train_time:74378ms step_avg:151.48ms
step:502/1480 train_time:74526ms step_avg:151.48ms
step:503/1480 train_time:74679ms step_avg:151.48ms
step:504/1480 train_time:74830ms step_avg:151.48ms
step:505/1480 train_time:74983ms step_avg:151.48ms
step:506/1480 train_time:75135ms step_avg:151.48ms
step:507/1480 train_time:75288ms step_avg:151.48ms
step:508/1480 train_time:75441ms step_avg:151.49ms
step:509/1480 train_time:75595ms step_avg:151.49ms
step:510/1480 train_time:75748ms step_avg:151.50ms
step:511/1480 train_time:75901ms step_avg:151.50ms
step:512/1480 train_time:76054ms step_avg:151.50ms
step:513/1480 train_time:76207ms step_avg:151.51ms
step:514/1480 train_time:76360ms step_avg:151.51ms
step:515/1480 train_time:76513ms step_avg:151.51ms
step:516/1480 train_time:76667ms step_avg:151.52ms
step:517/1480 train_time:76820ms step_avg:151.52ms
step:518/1480 train_time:76973ms step_avg:151.52ms
step:519/1480 train_time:77127ms step_avg:151.53ms
step:520/1480 train_time:77280ms step_avg:151.53ms
step:521/1480 train_time:77433ms step_avg:151.53ms
step:522/1480 train_time:77586ms step_avg:151.54ms
step:523/1480 train_time:77739ms step_avg:151.54ms
step:524/1480 train_time:77892ms step_avg:151.54ms
step:525/1480 train_time:78046ms step_avg:151.55ms
step:526/1480 train_time:78199ms step_avg:151.55ms
step:527/1480 train_time:78353ms step_avg:151.55ms
step:528/1480 train_time:78507ms step_avg:151.56ms
step:529/1480 train_time:78658ms step_avg:151.56ms
step:530/1480 train_time:78811ms step_avg:151.56ms
step:531/1480 train_time:78964ms step_avg:151.56ms
step:532/1480 train_time:79117ms step_avg:151.56ms
step:533/1480 train_time:79269ms step_avg:151.57ms
step:534/1480 train_time:79422ms step_avg:151.57ms
step:535/1480 train_time:79575ms step_avg:151.57ms
step:536/1480 train_time:79727ms step_avg:151.57ms
step:537/1480 train_time:79881ms step_avg:151.58ms
step:538/1480 train_time:80034ms step_avg:151.58ms
step:539/1480 train_time:80187ms step_avg:151.58ms
step:540/1480 train_time:80339ms step_avg:151.58ms
step:541/1480 train_time:80492ms step_avg:151.58ms
step:542/1480 train_time:80645ms step_avg:151.59ms
step:543/1480 train_time:80799ms step_avg:151.59ms
step:544/1480 train_time:80951ms step_avg:151.59ms
step:545/1480 train_time:81105ms step_avg:151.60ms
step:546/1480 train_time:81258ms step_avg:151.60ms
step:547/1480 train_time:81410ms step_avg:151.60ms
step:548/1480 train_time:81563ms step_avg:151.60ms
step:549/1480 train_time:81715ms step_avg:151.60ms
step:550/1480 train_time:81868ms step_avg:151.61ms
step:551/1480 train_time:82024ms step_avg:151.62ms
step:552/1480 train_time:82179ms step_avg:151.62ms
step:553/1480 train_time:82335ms step_avg:151.63ms
step:554/1480 train_time:82489ms step_avg:151.63ms
step:555/1480 train_time:82644ms step_avg:151.64ms
step:556/1480 train_time:82798ms step_avg:151.65ms
step:557/1480 train_time:82954ms step_avg:151.65ms
step:558/1480 train_time:83109ms step_avg:151.66ms
step:559/1480 train_time:83264ms step_avg:151.66ms
step:560/1480 train_time:83419ms step_avg:151.67ms
step:561/1480 train_time:83576ms step_avg:151.68ms
step:562/1480 train_time:83730ms step_avg:151.68ms
step:563/1480 train_time:83884ms step_avg:151.69ms
step:564/1480 train_time:84040ms step_avg:151.70ms
step:565/1480 train_time:84195ms step_avg:151.70ms
step:566/1480 train_time:84351ms step_avg:151.71ms
step:567/1480 train_time:84506ms step_avg:151.72ms
step:568/1480 train_time:84660ms step_avg:151.72ms
step:569/1480 train_time:84831ms step_avg:151.75ms
step:570/1480 train_time:84968ms step_avg:151.73ms
step:571/1480 train_time:85123ms step_avg:151.73ms
step:572/1480 train_time:85277ms step_avg:151.74ms
step:573/1480 train_time:85431ms step_avg:151.74ms
step:574/1480 train_time:85588ms step_avg:151.75ms
step:575/1480 train_time:85743ms step_avg:151.76ms
step:576/1480 train_time:85897ms step_avg:151.76ms
step:577/1480 train_time:86052ms step_avg:151.77ms
step:578/1480 train_time:86207ms step_avg:151.77ms
step:579/1480 train_time:86361ms step_avg:151.78ms
step:580/1480 train_time:86515ms step_avg:151.78ms
step:581/1480 train_time:86669ms step_avg:151.78ms
step:582/1480 train_time:86824ms step_avg:151.79ms
step:583/1480 train_time:86978ms step_avg:151.79ms
step:584/1480 train_time:87132ms step_avg:151.80ms
step:585/1480 train_time:87288ms step_avg:151.80ms
step:586/1480 train_time:87443ms step_avg:151.81ms
step:587/1480 train_time:87598ms step_avg:151.82ms
step:588/1480 train_time:87752ms step_avg:151.82ms
step:589/1480 train_time:87908ms step_avg:151.83ms
step:590/1480 train_time:88062ms step_avg:151.83ms
step:591/1480 train_time:88216ms step_avg:151.84ms
step:592/1480 train_time:88370ms step_avg:151.84ms
step:593/1480 train_time:88526ms step_avg:151.85ms
step:594/1480 train_time:88681ms step_avg:151.85ms
step:595/1480 train_time:88836ms step_avg:151.86ms
step:596/1480 train_time:88993ms step_avg:151.87ms
step:597/1480 train_time:89148ms step_avg:151.87ms
step:598/1480 train_time:89304ms step_avg:151.88ms
step:599/1480 train_time:89458ms step_avg:151.88ms
step:600/1480 train_time:89612ms step_avg:151.89ms
step:601/1480 train_time:89767ms step_avg:151.89ms
step:602/1480 train_time:89924ms step_avg:151.90ms
step:603/1480 train_time:90079ms step_avg:151.90ms
step:604/1480 train_time:90234ms step_avg:151.91ms
step:605/1480 train_time:90389ms step_avg:151.91ms
step:606/1480 train_time:90544ms step_avg:151.92ms
step:607/1480 train_time:90699ms step_avg:151.92ms
step:608/1480 train_time:90854ms step_avg:151.93ms
step:609/1480 train_time:91008ms step_avg:151.93ms
step:610/1480 train_time:91162ms step_avg:151.94ms
step:611/1480 train_time:91316ms step_avg:151.94ms
step:612/1480 train_time:91470ms step_avg:151.94ms
step:613/1480 train_time:91627ms step_avg:151.95ms
step:614/1480 train_time:91783ms step_avg:151.96ms
step:615/1480 train_time:91937ms step_avg:151.96ms
step:616/1480 train_time:92090ms step_avg:151.96ms
step:617/1480 train_time:92246ms step_avg:151.97ms
step:618/1480 train_time:92401ms step_avg:151.98ms
step:619/1480 train_time:92556ms step_avg:151.98ms
step:620/1480 train_time:92712ms step_avg:151.99ms
step:621/1480 train_time:92866ms step_avg:151.99ms
step:622/1480 train_time:93022ms step_avg:152.00ms
step:623/1480 train_time:93177ms step_avg:152.00ms
step:624/1480 train_time:93331ms step_avg:152.01ms
step:625/1480 train_time:93486ms step_avg:152.01ms
step:625/1480 val_loss:3.6129 train_time:93555ms step_avg:152.12ms
step:626/1480 train_time:93646ms step_avg:152.02ms
step:627/1480 train_time:93801ms step_avg:152.03ms
step:628/1480 train_time:93955ms step_avg:152.03ms
step:629/1480 train_time:94110ms step_avg:152.04ms
step:630/1480 train_time:94264ms step_avg:152.04ms
step:631/1480 train_time:94418ms step_avg:152.04ms
step:632/1480 train_time:94573ms step_avg:152.05ms
step:633/1480 train_time:94729ms step_avg:152.05ms
step:634/1480 train_time:94885ms step_avg:152.06ms
step:635/1480 train_time:95040ms step_avg:152.06ms
step:636/1480 train_time:95194ms step_avg:152.07ms
step:637/1480 train_time:95348ms step_avg:152.07ms
step:638/1480 train_time:95503ms step_avg:152.07ms
step:639/1480 train_time:95657ms step_avg:152.08ms
step:640/1480 train_time:95813ms step_avg:152.08ms
step:641/1480 train_time:95969ms step_avg:152.09ms
step:642/1480 train_time:96123ms step_avg:152.09ms
step:643/1480 train_time:96278ms step_avg:152.10ms
step:644/1480 train_time:96433ms step_avg:152.10ms
step:645/1480 train_time:96587ms step_avg:152.11ms
step:646/1480 train_time:96742ms step_avg:152.11ms
step:647/1480 train_time:96896ms step_avg:152.11ms
step:648/1480 train_time:97053ms step_avg:152.12ms
step:649/1480 train_time:97209ms step_avg:152.13ms
step:650/1480 train_time:97363ms step_avg:152.13ms
step:651/1480 train_time:97518ms step_avg:152.13ms
step:652/1480 train_time:97673ms step_avg:152.14ms
step:653/1480 train_time:97828ms step_avg:152.14ms
step:654/1480 train_time:97982ms step_avg:152.15ms
step:655/1480 train_time:98136ms step_avg:152.15ms
step:656/1480 train_time:98292ms step_avg:152.16ms
step:657/1480 train_time:98446ms step_avg:152.16ms
step:658/1480 train_time:98601ms step_avg:152.16ms
step:659/1480 train_time:98756ms step_avg:152.17ms
step:660/1480 train_time:98913ms step_avg:152.17ms
step:661/1480 train_time:99070ms step_avg:152.18ms
step:662/1480 train_time:99226ms step_avg:152.19ms
step:663/1480 train_time:99381ms step_avg:152.19ms
step:664/1480 train_time:99537ms step_avg:152.20ms
step:665/1480 train_time:99694ms step_avg:152.20ms
step:666/1480 train_time:99850ms step_avg:152.21ms
step:667/1480 train_time:100008ms step_avg:152.22ms
step:668/1480 train_time:100164ms step_avg:152.23ms
step:669/1480 train_time:100322ms step_avg:152.23ms
step:670/1480 train_time:100478ms step_avg:152.24ms
step:671/1480 train_time:100634ms step_avg:152.24ms
step:672/1480 train_time:100791ms step_avg:152.25ms
step:673/1480 train_time:100947ms step_avg:152.26ms
step:674/1480 train_time:101104ms step_avg:152.26ms
step:675/1480 train_time:101260ms step_avg:152.27ms
step:676/1480 train_time:101416ms step_avg:152.28ms
step:677/1480 train_time:101573ms step_avg:152.28ms
step:678/1480 train_time:101730ms step_avg:152.29ms
step:679/1480 train_time:101886ms step_avg:152.30ms
step:680/1480 train_time:102044ms step_avg:152.30ms
step:681/1480 train_time:102198ms step_avg:152.31ms
step:682/1480 train_time:102355ms step_avg:152.31ms
step:683/1480 train_time:102512ms step_avg:152.32ms
step:684/1480 train_time:102668ms step_avg:152.33ms
step:685/1480 train_time:102823ms step_avg:152.33ms
step:686/1480 train_time:102981ms step_avg:152.34ms
step:687/1480 train_time:103136ms step_avg:152.34ms
step:688/1480 train_time:103294ms step_avg:152.35ms
step:689/1480 train_time:103452ms step_avg:152.36ms
step:690/1480 train_time:103611ms step_avg:152.37ms
step:691/1480 train_time:103766ms step_avg:152.37ms
step:692/1480 train_time:103922ms step_avg:152.38ms
step:693/1480 train_time:104077ms step_avg:152.38ms
step:694/1480 train_time:104233ms step_avg:152.39ms
step:695/1480 train_time:104389ms step_avg:152.39ms
step:696/1480 train_time:104545ms step_avg:152.40ms
step:697/1480 train_time:104701ms step_avg:152.40ms
step:698/1480 train_time:104857ms step_avg:152.41ms
step:699/1480 train_time:105013ms step_avg:152.41ms
step:700/1480 train_time:105169ms step_avg:152.42ms
step:701/1480 train_time:105324ms step_avg:152.42ms
step:702/1480 train_time:105480ms step_avg:152.43ms
step:703/1480 train_time:105636ms step_avg:152.43ms
step:704/1480 train_time:105793ms step_avg:152.44ms
step:705/1480 train_time:105949ms step_avg:152.44ms
step:706/1480 train_time:106108ms step_avg:152.45ms
step:707/1480 train_time:106265ms step_avg:152.46ms
step:708/1480 train_time:106421ms step_avg:152.47ms
step:709/1480 train_time:106577ms step_avg:152.47ms
step:710/1480 train_time:106733ms step_avg:152.48ms
step:711/1480 train_time:106890ms step_avg:152.48ms
step:712/1480 train_time:107046ms step_avg:152.49ms
step:713/1480 train_time:107203ms step_avg:152.49ms
step:714/1480 train_time:107360ms step_avg:152.50ms
step:715/1480 train_time:107515ms step_avg:152.50ms
step:716/1480 train_time:107671ms step_avg:152.51ms
step:717/1480 train_time:107828ms step_avg:152.52ms
step:718/1480 train_time:107984ms step_avg:152.52ms
step:719/1480 train_time:108140ms step_avg:152.52ms
step:720/1480 train_time:108298ms step_avg:152.53ms
step:721/1480 train_time:108455ms step_avg:152.54ms
step:722/1480 train_time:108612ms step_avg:152.54ms
step:723/1480 train_time:108768ms step_avg:152.55ms
step:724/1480 train_time:108924ms step_avg:152.55ms
step:725/1480 train_time:109080ms step_avg:152.56ms
step:726/1480 train_time:109237ms step_avg:152.57ms
step:727/1480 train_time:109395ms step_avg:152.57ms
step:728/1480 train_time:109549ms step_avg:152.58ms
step:729/1480 train_time:109708ms step_avg:152.58ms
step:730/1480 train_time:109865ms step_avg:152.59ms
step:731/1480 train_time:110022ms step_avg:152.60ms
step:732/1480 train_time:110177ms step_avg:152.60ms
step:733/1480 train_time:110333ms step_avg:152.60ms
step:734/1480 train_time:110492ms step_avg:152.61ms
step:735/1480 train_time:110648ms step_avg:152.62ms
step:736/1480 train_time:110804ms step_avg:152.62ms
step:737/1480 train_time:110960ms step_avg:152.63ms
step:738/1480 train_time:111115ms step_avg:152.63ms
step:739/1480 train_time:111273ms step_avg:152.64ms
step:740/1480 train_time:111431ms step_avg:152.65ms
step:741/1480 train_time:111588ms step_avg:152.65ms
step:742/1480 train_time:111744ms step_avg:152.66ms
step:743/1480 train_time:111900ms step_avg:152.66ms
step:744/1480 train_time:112055ms step_avg:152.66ms
step:745/1480 train_time:112213ms step_avg:152.67ms
step:746/1480 train_time:112369ms step_avg:152.67ms
step:747/1480 train_time:112525ms step_avg:152.68ms
step:748/1480 train_time:112685ms step_avg:152.69ms
step:749/1480 train_time:112842ms step_avg:152.70ms
step:750/1480 train_time:112997ms step_avg:152.70ms
step:750/1480 val_loss:3.5569 train_time:113068ms step_avg:152.79ms
step:751/1480 train_time:113161ms step_avg:152.71ms
step:752/1480 train_time:113318ms step_avg:152.72ms
step:753/1480 train_time:113473ms step_avg:152.72ms
step:754/1480 train_time:113628ms step_avg:152.73ms
step:755/1480 train_time:113786ms step_avg:152.73ms
step:756/1480 train_time:113942ms step_avg:152.74ms
step:757/1480 train_time:114100ms step_avg:152.74ms
step:758/1480 train_time:114258ms step_avg:152.75ms
step:759/1480 train_time:114429ms step_avg:152.78ms
step:760/1480 train_time:114571ms step_avg:152.76ms
step:761/1480 train_time:114728ms step_avg:152.77ms
step:762/1480 train_time:114884ms step_avg:152.77ms
step:763/1480 train_time:115041ms step_avg:152.78ms
step:764/1480 train_time:115197ms step_avg:152.78ms
step:765/1480 train_time:115353ms step_avg:152.79ms
step:766/1480 train_time:115512ms step_avg:152.79ms
step:767/1480 train_time:115668ms step_avg:152.80ms
step:768/1480 train_time:115824ms step_avg:152.80ms
step:769/1480 train_time:115983ms step_avg:152.81ms
step:770/1480 train_time:116140ms step_avg:152.82ms
step:771/1480 train_time:116297ms step_avg:152.82ms
step:772/1480 train_time:116454ms step_avg:152.83ms
step:773/1480 train_time:116612ms step_avg:152.83ms
step:774/1480 train_time:116770ms step_avg:152.84ms
step:775/1480 train_time:116927ms step_avg:152.85ms
step:776/1480 train_time:117087ms step_avg:152.86ms
step:777/1480 train_time:117247ms step_avg:152.86ms
step:778/1480 train_time:117405ms step_avg:152.87ms
step:779/1480 train_time:117563ms step_avg:152.88ms
step:780/1480 train_time:117721ms step_avg:152.88ms
step:781/1480 train_time:117880ms step_avg:152.89ms
step:782/1480 train_time:118038ms step_avg:152.90ms
step:783/1480 train_time:118196ms step_avg:152.91ms
step:784/1480 train_time:118353ms step_avg:152.91ms
step:785/1480 train_time:118510ms step_avg:152.92ms
step:786/1480 train_time:118667ms step_avg:152.92ms
step:787/1480 train_time:118825ms step_avg:152.93ms
step:788/1480 train_time:118987ms step_avg:152.94ms
step:789/1480 train_time:119144ms step_avg:152.95ms
step:790/1480 train_time:119301ms step_avg:152.95ms
step:791/1480 train_time:119461ms step_avg:152.96ms
step:792/1480 train_time:119619ms step_avg:152.97ms
step:793/1480 train_time:119776ms step_avg:152.97ms
step:794/1480 train_time:119933ms step_avg:152.98ms
step:795/1480 train_time:120095ms step_avg:152.99ms
step:796/1480 train_time:120253ms step_avg:152.99ms
step:797/1480 train_time:120411ms step_avg:153.00ms
step:798/1480 train_time:120571ms step_avg:153.01ms
step:799/1480 train_time:120732ms step_avg:153.02ms
step:800/1480 train_time:120889ms step_avg:153.02ms
step:801/1480 train_time:121048ms step_avg:153.03ms
step:802/1480 train_time:121207ms step_avg:153.04ms
step:803/1480 train_time:121365ms step_avg:153.04ms
step:804/1480 train_time:121522ms step_avg:153.05ms
step:805/1480 train_time:121681ms step_avg:153.06ms
step:806/1480 train_time:121838ms step_avg:153.06ms
step:807/1480 train_time:121994ms step_avg:153.07ms
step:808/1480 train_time:122151ms step_avg:153.07ms
step:809/1480 train_time:122308ms step_avg:153.08ms
step:810/1480 train_time:122465ms step_avg:153.08ms
step:811/1480 train_time:122622ms step_avg:153.09ms
step:812/1480 train_time:122780ms step_avg:153.09ms
step:813/1480 train_time:122937ms step_avg:153.10ms
step:814/1480 train_time:123094ms step_avg:153.10ms
step:815/1480 train_time:123251ms step_avg:153.11ms
step:816/1480 train_time:123410ms step_avg:153.11ms
step:817/1480 train_time:123569ms step_avg:153.12ms
step:818/1480 train_time:123725ms step_avg:153.12ms
step:819/1480 train_time:123884ms step_avg:153.13ms
step:820/1480 train_time:124041ms step_avg:153.14ms
step:821/1480 train_time:124198ms step_avg:153.14ms
step:822/1480 train_time:124355ms step_avg:153.15ms
step:823/1480 train_time:124512ms step_avg:153.15ms
step:824/1480 train_time:124669ms step_avg:153.16ms
step:825/1480 train_time:124828ms step_avg:153.16ms
step:826/1480 train_time:124987ms step_avg:153.17ms
step:827/1480 train_time:125146ms step_avg:153.18ms
step:828/1480 train_time:125303ms step_avg:153.18ms
step:829/1480 train_time:125464ms step_avg:153.19ms
step:830/1480 train_time:125624ms step_avg:153.20ms
step:831/1480 train_time:125782ms step_avg:153.21ms
step:832/1480 train_time:125940ms step_avg:153.21ms
step:833/1480 train_time:126097ms step_avg:153.22ms
step:834/1480 train_time:126255ms step_avg:153.22ms
step:835/1480 train_time:126414ms step_avg:153.23ms
step:836/1480 train_time:126573ms step_avg:153.24ms
step:837/1480 train_time:126729ms step_avg:153.24ms
step:838/1480 train_time:126888ms step_avg:153.25ms
step:839/1480 train_time:127045ms step_avg:153.25ms
step:840/1480 train_time:127202ms step_avg:153.25ms
step:841/1480 train_time:127359ms step_avg:153.26ms
step:842/1480 train_time:127517ms step_avg:153.27ms
step:843/1480 train_time:127674ms step_avg:153.27ms
step:844/1480 train_time:127831ms step_avg:153.27ms
step:845/1480 train_time:127988ms step_avg:153.28ms
step:846/1480 train_time:128147ms step_avg:153.29ms
step:847/1480 train_time:128304ms step_avg:153.29ms
step:848/1480 train_time:128463ms step_avg:153.30ms
step:849/1480 train_time:128621ms step_avg:153.30ms
step:850/1480 train_time:128778ms step_avg:153.31ms
step:851/1480 train_time:128938ms step_avg:153.31ms
step:852/1480 train_time:129095ms step_avg:153.32ms
step:853/1480 train_time:129252ms step_avg:153.32ms
step:854/1480 train_time:129410ms step_avg:153.33ms
step:855/1480 train_time:129568ms step_avg:153.34ms
step:856/1480 train_time:129725ms step_avg:153.34ms
step:857/1480 train_time:129886ms step_avg:153.35ms
step:858/1480 train_time:130045ms step_avg:153.36ms
step:859/1480 train_time:130205ms step_avg:153.36ms
step:860/1480 train_time:130364ms step_avg:153.37ms
step:861/1480 train_time:130523ms step_avg:153.38ms
step:862/1480 train_time:130687ms step_avg:153.39ms
step:863/1480 train_time:130846ms step_avg:153.39ms
step:864/1480 train_time:131004ms step_avg:153.40ms
step:865/1480 train_time:131162ms step_avg:153.41ms
step:866/1480 train_time:131320ms step_avg:153.41ms
step:867/1480 train_time:131479ms step_avg:153.42ms
step:868/1480 train_time:131637ms step_avg:153.42ms
step:869/1480 train_time:131794ms step_avg:153.43ms
step:870/1480 train_time:131952ms step_avg:153.43ms
step:871/1480 train_time:132109ms step_avg:153.44ms
step:872/1480 train_time:132267ms step_avg:153.44ms
step:873/1480 train_time:132423ms step_avg:153.45ms
step:874/1480 train_time:132584ms step_avg:153.45ms
step:875/1480 train_time:132745ms step_avg:153.46ms
step:875/1480 val_loss:3.5094 train_time:132816ms step_avg:153.54ms
step:876/1480 train_time:132906ms step_avg:153.47ms
step:877/1480 train_time:133062ms step_avg:153.47ms
step:878/1480 train_time:133221ms step_avg:153.48ms
step:879/1480 train_time:133380ms step_avg:153.49ms
step:880/1480 train_time:133538ms step_avg:153.49ms
step:881/1480 train_time:133695ms step_avg:153.50ms
step:882/1480 train_time:133855ms step_avg:153.50ms
step:883/1480 train_time:134015ms step_avg:153.51ms
step:884/1480 train_time:134176ms step_avg:153.52ms
step:885/1480 train_time:134336ms step_avg:153.53ms
step:886/1480 train_time:134495ms step_avg:153.53ms
step:887/1480 train_time:134654ms step_avg:153.54ms
step:888/1480 train_time:134819ms step_avg:153.55ms
step:889/1480 train_time:134980ms step_avg:153.56ms
step:890/1480 train_time:135138ms step_avg:153.57ms
step:891/1480 train_time:135296ms step_avg:153.57ms
step:892/1480 train_time:135456ms step_avg:153.58ms
step:893/1480 train_time:135615ms step_avg:153.58ms
step:894/1480 train_time:135773ms step_avg:153.59ms
step:895/1480 train_time:135934ms step_avg:153.60ms
step:896/1480 train_time:136091ms step_avg:153.60ms
step:897/1480 train_time:136254ms step_avg:153.61ms
step:898/1480 train_time:136414ms step_avg:153.62ms
step:899/1480 train_time:136574ms step_avg:153.63ms
step:900/1480 train_time:136733ms step_avg:153.63ms
step:901/1480 train_time:136892ms step_avg:153.64ms
step:902/1480 train_time:137049ms step_avg:153.64ms
step:903/1480 train_time:137213ms step_avg:153.65ms
step:904/1480 train_time:137372ms step_avg:153.66ms
step:905/1480 train_time:137531ms step_avg:153.67ms
step:906/1480 train_time:137691ms step_avg:153.67ms
step:907/1480 train_time:137855ms step_avg:153.68ms
step:908/1480 train_time:138012ms step_avg:153.69ms
step:909/1480 train_time:138170ms step_avg:153.69ms
step:910/1480 train_time:138334ms step_avg:153.70ms
step:911/1480 train_time:138494ms step_avg:153.71ms
step:912/1480 train_time:138654ms step_avg:153.72ms
step:913/1480 train_time:138816ms step_avg:153.73ms
step:914/1480 train_time:138977ms step_avg:153.74ms
step:915/1480 train_time:139139ms step_avg:153.75ms
step:916/1480 train_time:139298ms step_avg:153.75ms
step:917/1480 train_time:139456ms step_avg:153.76ms
step:918/1480 train_time:139619ms step_avg:153.77ms
step:919/1480 train_time:139780ms step_avg:153.77ms
step:920/1480 train_time:139939ms step_avg:153.78ms
step:921/1480 train_time:140097ms step_avg:153.78ms
step:922/1480 train_time:140260ms step_avg:153.79ms
step:923/1480 train_time:140417ms step_avg:153.80ms
step:924/1480 train_time:140578ms step_avg:153.81ms
step:925/1480 train_time:140737ms step_avg:153.81ms
step:926/1480 train_time:140894ms step_avg:153.81ms
step:927/1480 train_time:141053ms step_avg:153.82ms
step:928/1480 train_time:141212ms step_avg:153.83ms
step:929/1480 train_time:141370ms step_avg:153.83ms
step:930/1480 train_time:141530ms step_avg:153.84ms
step:931/1480 train_time:141690ms step_avg:153.84ms
step:932/1480 train_time:141849ms step_avg:153.85ms
step:933/1480 train_time:142010ms step_avg:153.86ms
step:934/1480 train_time:142168ms step_avg:153.86ms
step:935/1480 train_time:142333ms step_avg:153.87ms
step:936/1480 train_time:142491ms step_avg:153.88ms
step:937/1480 train_time:142652ms step_avg:153.89ms
step:938/1480 train_time:142809ms step_avg:153.89ms
step:939/1480 train_time:142970ms step_avg:153.90ms
step:940/1480 train_time:143132ms step_avg:153.91ms
step:941/1480 train_time:143291ms step_avg:153.91ms
step:942/1480 train_time:143450ms step_avg:153.92ms
step:943/1480 train_time:143611ms step_avg:153.92ms
step:944/1480 train_time:143774ms step_avg:153.93ms
step:945/1480 train_time:143933ms step_avg:153.94ms
step:946/1480 train_time:144093ms step_avg:153.95ms
step:947/1480 train_time:144255ms step_avg:153.95ms
step:948/1480 train_time:144415ms step_avg:153.96ms
step:949/1480 train_time:144589ms step_avg:153.98ms
step:950/1480 train_time:144735ms step_avg:153.97ms
step:951/1480 train_time:144895ms step_avg:153.98ms
step:952/1480 train_time:145054ms step_avg:153.99ms
step:953/1480 train_time:145214ms step_avg:153.99ms
step:954/1480 train_time:145377ms step_avg:154.00ms
step:955/1480 train_time:145535ms step_avg:154.01ms
step:956/1480 train_time:145694ms step_avg:154.01ms
step:957/1480 train_time:145857ms step_avg:154.02ms
step:958/1480 train_time:146021ms step_avg:154.03ms
step:959/1480 train_time:146180ms step_avg:154.04ms
step:960/1480 train_time:146339ms step_avg:154.04ms
step:961/1480 train_time:146498ms step_avg:154.05ms
step:962/1480 train_time:146658ms step_avg:154.05ms
step:963/1480 train_time:146818ms step_avg:154.06ms
step:964/1480 train_time:146979ms step_avg:154.07ms
step:965/1480 train_time:147137ms step_avg:154.07ms
step:966/1480 train_time:147295ms step_avg:154.07ms
step:967/1480 train_time:147455ms step_avg:154.08ms
step:968/1480 train_time:147614ms step_avg:154.09ms
step:969/1480 train_time:147774ms step_avg:154.09ms
step:970/1480 train_time:147932ms step_avg:154.10ms
step:971/1480 train_time:148091ms step_avg:154.10ms
step:972/1480 train_time:148249ms step_avg:154.10ms
step:973/1480 train_time:148407ms step_avg:154.11ms
step:974/1480 train_time:148568ms step_avg:154.12ms
step:975/1480 train_time:148728ms step_avg:154.12ms
step:976/1480 train_time:148889ms step_avg:154.13ms
step:977/1480 train_time:149047ms step_avg:154.13ms
step:978/1480 train_time:149206ms step_avg:154.14ms
step:979/1480 train_time:149366ms step_avg:154.14ms
step:980/1480 train_time:149525ms step_avg:154.15ms
step:981/1480 train_time:149685ms step_avg:154.16ms
step:982/1480 train_time:149844ms step_avg:154.16ms
step:983/1480 train_time:150003ms step_avg:154.17ms
step:984/1480 train_time:150162ms step_avg:154.17ms
step:985/1480 train_time:150322ms step_avg:154.18ms
step:986/1480 train_time:150481ms step_avg:154.18ms
step:987/1480 train_time:150639ms step_avg:154.19ms
step:988/1480 train_time:150797ms step_avg:154.19ms
step:989/1480 train_time:150957ms step_avg:154.20ms
step:990/1480 train_time:151118ms step_avg:154.20ms
step:991/1480 train_time:151279ms step_avg:154.21ms
step:992/1480 train_time:151445ms step_avg:154.22ms
step:993/1480 train_time:151615ms step_avg:154.24ms
step:994/1480 train_time:151775ms step_avg:154.24ms
step:995/1480 train_time:151934ms step_avg:154.25ms
step:996/1480 train_time:152092ms step_avg:154.25ms
step:997/1480 train_time:152252ms step_avg:154.26ms
step:998/1480 train_time:152412ms step_avg:154.26ms
step:999/1480 train_time:152571ms step_avg:154.27ms
step:1000/1480 train_time:152732ms step_avg:154.27ms
step:1000/1480 val_loss:3.4467 train_time:152805ms step_avg:154.35ms
step:1001/1480 train_time:152897ms step_avg:154.29ms
step:1002/1480 train_time:153056ms step_avg:154.29ms
step:1003/1480 train_time:153219ms step_avg:154.30ms
step:1004/1480 train_time:153382ms step_avg:154.31ms
step:1005/1480 train_time:153543ms step_avg:154.31ms
step:1006/1480 train_time:153704ms step_avg:154.32ms
step:1007/1480 train_time:153864ms step_avg:154.33ms
step:1008/1480 train_time:154025ms step_avg:154.33ms
step:1009/1480 train_time:154191ms step_avg:154.35ms
step:1010/1480 train_time:154350ms step_avg:154.35ms
step:1011/1480 train_time:154510ms step_avg:154.36ms
step:1012/1480 train_time:154669ms step_avg:154.36ms
step:1013/1480 train_time:154830ms step_avg:154.37ms
step:1014/1480 train_time:154992ms step_avg:154.37ms
step:1015/1480 train_time:155154ms step_avg:154.38ms
step:1016/1480 train_time:155315ms step_avg:154.39ms
step:1017/1480 train_time:155476ms step_avg:154.40ms
step:1018/1480 train_time:155637ms step_avg:154.40ms
step:1019/1480 train_time:155797ms step_avg:154.41ms
step:1020/1480 train_time:155957ms step_avg:154.41ms
step:1021/1480 train_time:156118ms step_avg:154.42ms
step:1022/1480 train_time:156278ms step_avg:154.42ms
step:1023/1480 train_time:156441ms step_avg:154.43ms
step:1024/1480 train_time:156601ms step_avg:154.44ms
step:1025/1480 train_time:156764ms step_avg:154.45ms
step:1026/1480 train_time:156924ms step_avg:154.45ms
step:1027/1480 train_time:157085ms step_avg:154.46ms
step:1028/1480 train_time:157249ms step_avg:154.47ms
step:1029/1480 train_time:157413ms step_avg:154.48ms
step:1030/1480 train_time:157573ms step_avg:154.48ms
step:1031/1480 train_time:157732ms step_avg:154.49ms
step:1032/1480 train_time:157898ms step_avg:154.50ms
step:1033/1480 train_time:158057ms step_avg:154.50ms
step:1034/1480 train_time:158216ms step_avg:154.51ms
step:1035/1480 train_time:158378ms step_avg:154.51ms
step:1036/1480 train_time:158539ms step_avg:154.52ms
step:1037/1480 train_time:158699ms step_avg:154.53ms
step:1038/1480 train_time:158859ms step_avg:154.53ms
step:1039/1480 train_time:159021ms step_avg:154.54ms
step:1040/1480 train_time:159182ms step_avg:154.55ms
step:1041/1480 train_time:159345ms step_avg:154.55ms
step:1042/1480 train_time:159505ms step_avg:154.56ms
step:1043/1480 train_time:159664ms step_avg:154.56ms
step:1044/1480 train_time:159823ms step_avg:154.57ms
step:1045/1480 train_time:159984ms step_avg:154.57ms
step:1046/1480 train_time:160145ms step_avg:154.58ms
step:1047/1480 train_time:160305ms step_avg:154.59ms
step:1048/1480 train_time:160466ms step_avg:154.59ms
step:1049/1480 train_time:160627ms step_avg:154.60ms
step:1050/1480 train_time:160787ms step_avg:154.60ms
step:1051/1480 train_time:160948ms step_avg:154.61ms
step:1052/1480 train_time:161110ms step_avg:154.62ms
step:1053/1480 train_time:161272ms step_avg:154.62ms
step:1054/1480 train_time:161433ms step_avg:154.63ms
step:1055/1480 train_time:161593ms step_avg:154.63ms
step:1056/1480 train_time:161750ms step_avg:154.64ms
step:1057/1480 train_time:161911ms step_avg:154.64ms
step:1058/1480 train_time:162072ms step_avg:154.65ms
step:1059/1480 train_time:162234ms step_avg:154.66ms
step:1060/1480 train_time:162394ms step_avg:154.66ms
step:1061/1480 train_time:162553ms step_avg:154.67ms
step:1062/1480 train_time:162714ms step_avg:154.67ms
step:1063/1480 train_time:162874ms step_avg:154.68ms
step:1064/1480 train_time:163032ms step_avg:154.68ms
step:1065/1480 train_time:163194ms step_avg:154.69ms
step:1066/1480 train_time:163355ms step_avg:154.69ms
step:1067/1480 train_time:163517ms step_avg:154.70ms
step:1068/1480 train_time:163676ms step_avg:154.70ms
step:1069/1480 train_time:163843ms step_avg:154.71ms
step:1070/1480 train_time:164003ms step_avg:154.72ms
step:1071/1480 train_time:164167ms step_avg:154.73ms
step:1072/1480 train_time:164326ms step_avg:154.73ms
step:1073/1480 train_time:164485ms step_avg:154.74ms
step:1074/1480 train_time:164644ms step_avg:154.74ms
step:1075/1480 train_time:164807ms step_avg:154.75ms
step:1076/1480 train_time:164967ms step_avg:154.75ms
step:1077/1480 train_time:165126ms step_avg:154.76ms
step:1078/1480 train_time:165292ms step_avg:154.77ms
step:1079/1480 train_time:165455ms step_avg:154.78ms
step:1080/1480 train_time:165616ms step_avg:154.78ms
step:1081/1480 train_time:165775ms step_avg:154.79ms
step:1082/1480 train_time:165935ms step_avg:154.79ms
step:1083/1480 train_time:166095ms step_avg:154.79ms
step:1084/1480 train_time:166255ms step_avg:154.80ms
step:1085/1480 train_time:166415ms step_avg:154.80ms
step:1086/1480 train_time:166576ms step_avg:154.81ms
step:1087/1480 train_time:166737ms step_avg:154.82ms
step:1088/1480 train_time:166898ms step_avg:154.82ms
step:1089/1480 train_time:167061ms step_avg:154.83ms
step:1090/1480 train_time:167223ms step_avg:154.84ms
step:1091/1480 train_time:167385ms step_avg:154.84ms
step:1092/1480 train_time:167545ms step_avg:154.85ms
step:1093/1480 train_time:167707ms step_avg:154.85ms
step:1094/1480 train_time:167868ms step_avg:154.86ms
step:1095/1480 train_time:168028ms step_avg:154.86ms
step:1096/1480 train_time:168189ms step_avg:154.87ms
step:1097/1480 train_time:168350ms step_avg:154.88ms
step:1098/1480 train_time:168512ms step_avg:154.88ms
step:1099/1480 train_time:168674ms step_avg:154.89ms
step:1100/1480 train_time:168839ms step_avg:154.90ms
step:1101/1480 train_time:169001ms step_avg:154.90ms
step:1102/1480 train_time:169164ms step_avg:154.91ms
step:1103/1480 train_time:169330ms step_avg:154.92ms
step:1104/1480 train_time:169491ms step_avg:154.93ms
step:1105/1480 train_time:169651ms step_avg:154.93ms
step:1106/1480 train_time:169813ms step_avg:154.94ms
step:1107/1480 train_time:169974ms step_avg:154.94ms
step:1108/1480 train_time:170133ms step_avg:154.95ms
step:1109/1480 train_time:170294ms step_avg:154.95ms
step:1110/1480 train_time:170454ms step_avg:154.96ms
step:1111/1480 train_time:170616ms step_avg:154.96ms
step:1112/1480 train_time:170778ms step_avg:154.97ms
step:1113/1480 train_time:170943ms step_avg:154.98ms
step:1114/1480 train_time:171107ms step_avg:154.99ms
step:1115/1480 train_time:171270ms step_avg:155.00ms
step:1116/1480 train_time:171430ms step_avg:155.00ms
step:1117/1480 train_time:171591ms step_avg:155.01ms
step:1118/1480 train_time:171754ms step_avg:155.01ms
step:1119/1480 train_time:171915ms step_avg:155.02ms
step:1120/1480 train_time:172078ms step_avg:155.03ms
step:1121/1480 train_time:172241ms step_avg:155.03ms
step:1122/1480 train_time:172401ms step_avg:155.04ms
step:1123/1480 train_time:172560ms step_avg:155.04ms
step:1124/1480 train_time:172725ms step_avg:155.05ms
step:1125/1480 train_time:172887ms step_avg:155.06ms
step:1125/1480 val_loss:3.3910 train_time:172963ms step_avg:155.12ms
step:1126/1480 train_time:173058ms step_avg:155.07ms
step:1127/1480 train_time:173214ms step_avg:155.07ms
step:1128/1480 train_time:173374ms step_avg:155.07ms
step:1129/1480 train_time:173539ms step_avg:155.08ms
step:1130/1480 train_time:173700ms step_avg:155.09ms
step:1131/1480 train_time:173867ms step_avg:155.10ms
step:1132/1480 train_time:174027ms step_avg:155.10ms
step:1133/1480 train_time:174192ms step_avg:155.11ms
step:1134/1480 train_time:174356ms step_avg:155.12ms
step:1135/1480 train_time:174517ms step_avg:155.13ms
step:1136/1480 train_time:174680ms step_avg:155.13ms
step:1137/1480 train_time:174841ms step_avg:155.14ms
step:1138/1480 train_time:175003ms step_avg:155.14ms
step:1139/1480 train_time:175178ms step_avg:155.16ms
step:1140/1480 train_time:175327ms step_avg:155.16ms
step:1141/1480 train_time:175491ms step_avg:155.16ms
step:1142/1480 train_time:175652ms step_avg:155.17ms
step:1143/1480 train_time:175818ms step_avg:155.18ms
step:1144/1480 train_time:175978ms step_avg:155.18ms
step:1145/1480 train_time:176138ms step_avg:155.19ms
step:1146/1480 train_time:176300ms step_avg:155.19ms
step:1147/1480 train_time:176462ms step_avg:155.20ms
step:1148/1480 train_time:176623ms step_avg:155.20ms
step:1149/1480 train_time:176785ms step_avg:155.21ms
step:1150/1480 train_time:176945ms step_avg:155.22ms
step:1151/1480 train_time:177107ms step_avg:155.22ms
step:1152/1480 train_time:177271ms step_avg:155.23ms
step:1153/1480 train_time:177437ms step_avg:155.24ms
step:1154/1480 train_time:177599ms step_avg:155.24ms
step:1155/1480 train_time:177761ms step_avg:155.25ms
step:1156/1480 train_time:177927ms step_avg:155.26ms
step:1157/1480 train_time:178091ms step_avg:155.27ms
step:1158/1480 train_time:178251ms step_avg:155.27ms
step:1159/1480 train_time:178412ms step_avg:155.28ms
step:1160/1480 train_time:178573ms step_avg:155.28ms
step:1161/1480 train_time:178736ms step_avg:155.29ms
step:1162/1480 train_time:178900ms step_avg:155.30ms
step:1163/1480 train_time:179063ms step_avg:155.30ms
step:1164/1480 train_time:179225ms step_avg:155.31ms
step:1165/1480 train_time:179384ms step_avg:155.31ms
step:1166/1480 train_time:179546ms step_avg:155.32ms
step:1167/1480 train_time:179706ms step_avg:155.32ms
step:1168/1480 train_time:179866ms step_avg:155.33ms
step:1169/1480 train_time:180030ms step_avg:155.33ms
step:1170/1480 train_time:180191ms step_avg:155.34ms
step:1171/1480 train_time:180354ms step_avg:155.34ms
step:1172/1480 train_time:180513ms step_avg:155.35ms
step:1173/1480 train_time:180675ms step_avg:155.35ms
step:1174/1480 train_time:180845ms step_avg:155.36ms
step:1175/1480 train_time:181007ms step_avg:155.37ms
step:1176/1480 train_time:181171ms step_avg:155.38ms
step:1177/1480 train_time:181339ms step_avg:155.39ms
step:1178/1480 train_time:181499ms step_avg:155.39ms
step:1179/1480 train_time:181658ms step_avg:155.40ms
step:1180/1480 train_time:181826ms step_avg:155.41ms
step:1181/1480 train_time:181988ms step_avg:155.41ms
step:1182/1480 train_time:182148ms step_avg:155.42ms
step:1183/1480 train_time:182311ms step_avg:155.42ms
step:1184/1480 train_time:182472ms step_avg:155.43ms
step:1185/1480 train_time:182638ms step_avg:155.44ms
step:1186/1480 train_time:182801ms step_avg:155.44ms
step:1187/1480 train_time:182973ms step_avg:155.46ms
step:1188/1480 train_time:183133ms step_avg:155.46ms
step:1189/1480 train_time:183295ms step_avg:155.47ms
step:1190/1480 train_time:183456ms step_avg:155.47ms
step:1191/1480 train_time:183620ms step_avg:155.48ms
step:1192/1480 train_time:183781ms step_avg:155.48ms
step:1193/1480 train_time:183941ms step_avg:155.49ms
step:1194/1480 train_time:184101ms step_avg:155.49ms
step:1195/1480 train_time:184264ms step_avg:155.50ms
step:1196/1480 train_time:184435ms step_avg:155.51ms
step:1197/1480 train_time:184597ms step_avg:155.52ms
step:1198/1480 train_time:184765ms step_avg:155.53ms
step:1199/1480 train_time:184927ms step_avg:155.53ms
step:1200/1480 train_time:185089ms step_avg:155.54ms
step:1201/1480 train_time:185249ms step_avg:155.54ms
step:1202/1480 train_time:185418ms step_avg:155.55ms
step:1203/1480 train_time:185584ms step_avg:155.56ms
step:1204/1480 train_time:185748ms step_avg:155.57ms
step:1205/1480 train_time:185908ms step_avg:155.57ms
step:1206/1480 train_time:186069ms step_avg:155.58ms
step:1207/1480 train_time:186229ms step_avg:155.58ms
step:1208/1480 train_time:186390ms step_avg:155.58ms
step:1209/1480 train_time:186555ms step_avg:155.59ms
step:1210/1480 train_time:186721ms step_avg:155.60ms
step:1211/1480 train_time:186884ms step_avg:155.61ms
step:1212/1480 train_time:187045ms step_avg:155.61ms
step:1213/1480 train_time:187208ms step_avg:155.62ms
step:1214/1480 train_time:187374ms step_avg:155.63ms
step:1215/1480 train_time:187540ms step_avg:155.63ms
step:1216/1480 train_time:187701ms step_avg:155.64ms
step:1217/1480 train_time:187864ms step_avg:155.65ms
step:1218/1480 train_time:188026ms step_avg:155.65ms
step:1219/1480 train_time:188194ms step_avg:155.66ms
step:1220/1480 train_time:188358ms step_avg:155.67ms
step:1221/1480 train_time:188520ms step_avg:155.67ms
step:1222/1480 train_time:188680ms step_avg:155.68ms
step:1223/1480 train_time:188842ms step_avg:155.68ms
step:1224/1480 train_time:189007ms step_avg:155.69ms
step:1225/1480 train_time:189170ms step_avg:155.70ms
step:1226/1480 train_time:189335ms step_avg:155.70ms
step:1227/1480 train_time:189500ms step_avg:155.71ms
step:1228/1480 train_time:189662ms step_avg:155.72ms
step:1229/1480 train_time:189825ms step_avg:155.72ms
step:1230/1480 train_time:189992ms step_avg:155.73ms
step:1231/1480 train_time:190159ms step_avg:155.74ms
step:1232/1480 train_time:190323ms step_avg:155.75ms
step:1233/1480 train_time:190484ms step_avg:155.75ms
step:1234/1480 train_time:190645ms step_avg:155.76ms
step:1235/1480 train_time:190811ms step_avg:155.76ms
step:1236/1480 train_time:190972ms step_avg:155.77ms
step:1237/1480 train_time:191135ms step_avg:155.77ms
step:1238/1480 train_time:191307ms step_avg:155.79ms
step:1239/1480 train_time:191470ms step_avg:155.79ms
step:1240/1480 train_time:191637ms step_avg:155.80ms
step:1241/1480 train_time:191801ms step_avg:155.81ms
step:1242/1480 train_time:191963ms step_avg:155.81ms
step:1243/1480 train_time:192125ms step_avg:155.82ms
step:1244/1480 train_time:192285ms step_avg:155.82ms
step:1245/1480 train_time:192447ms step_avg:155.83ms
step:1246/1480 train_time:192610ms step_avg:155.83ms
step:1247/1480 train_time:192775ms step_avg:155.84ms
step:1248/1480 train_time:192938ms step_avg:155.85ms
step:1249/1480 train_time:193100ms step_avg:155.85ms
step:1250/1480 train_time:193262ms step_avg:155.86ms
step:1250/1480 val_loss:3.3404 train_time:193337ms step_avg:155.92ms
step:1251/1480 train_time:193431ms step_avg:155.87ms
step:1252/1480 train_time:193595ms step_avg:155.87ms
step:1253/1480 train_time:193755ms step_avg:155.88ms
step:1254/1480 train_time:193917ms step_avg:155.88ms
step:1255/1480 train_time:194086ms step_avg:155.89ms
step:1256/1480 train_time:194251ms step_avg:155.90ms
step:1257/1480 train_time:194413ms step_avg:155.90ms
step:1258/1480 train_time:194580ms step_avg:155.91ms
step:1259/1480 train_time:194742ms step_avg:155.92ms
step:1260/1480 train_time:194903ms step_avg:155.92ms
step:1261/1480 train_time:195066ms step_avg:155.93ms
step:1262/1480 train_time:195232ms step_avg:155.94ms
step:1263/1480 train_time:195398ms step_avg:155.94ms
step:1264/1480 train_time:195557ms step_avg:155.95ms
step:1265/1480 train_time:195717ms step_avg:155.95ms
step:1266/1480 train_time:195880ms step_avg:155.96ms
step:1267/1480 train_time:196041ms step_avg:155.96ms
step:1268/1480 train_time:196202ms step_avg:155.96ms
step:1269/1480 train_time:196368ms step_avg:155.97ms
step:1270/1480 train_time:196531ms step_avg:155.98ms
step:1271/1480 train_time:196694ms step_avg:155.98ms
step:1272/1480 train_time:196855ms step_avg:155.99ms
step:1273/1480 train_time:197017ms step_avg:155.99ms
step:1274/1480 train_time:197183ms step_avg:156.00ms
step:1275/1480 train_time:197343ms step_avg:156.00ms
step:1276/1480 train_time:197502ms step_avg:156.00ms
step:1277/1480 train_time:197665ms step_avg:156.01ms
step:1278/1480 train_time:197825ms step_avg:156.01ms
step:1279/1480 train_time:197987ms step_avg:156.02ms
step:1280/1480 train_time:198155ms step_avg:156.03ms
step:1281/1480 train_time:198317ms step_avg:156.03ms
step:1282/1480 train_time:198477ms step_avg:156.04ms
step:1283/1480 train_time:198639ms step_avg:156.04ms
step:1284/1480 train_time:198802ms step_avg:156.05ms
step:1285/1480 train_time:198964ms step_avg:156.05ms
step:1286/1480 train_time:199126ms step_avg:156.06ms
step:1287/1480 train_time:199290ms step_avg:156.06ms
step:1288/1480 train_time:199452ms step_avg:156.07ms
step:1289/1480 train_time:199622ms step_avg:156.08ms
step:1290/1480 train_time:199789ms step_avg:156.09ms
step:1291/1480 train_time:199954ms step_avg:156.09ms
step:1292/1480 train_time:200118ms step_avg:156.10ms
step:1293/1480 train_time:200283ms step_avg:156.11ms
step:1294/1480 train_time:200446ms step_avg:156.11ms
step:1295/1480 train_time:200609ms step_avg:156.12ms
step:1296/1480 train_time:200773ms step_avg:156.12ms
step:1297/1480 train_time:200936ms step_avg:156.13ms
step:1298/1480 train_time:201098ms step_avg:156.13ms
step:1299/1480 train_time:201261ms step_avg:156.14ms
step:1300/1480 train_time:201421ms step_avg:156.14ms
step:1301/1480 train_time:201582ms step_avg:156.14ms
step:1302/1480 train_time:201749ms step_avg:156.15ms
step:1303/1480 train_time:201918ms step_avg:156.16ms
step:1304/1480 train_time:202083ms step_avg:156.17ms
step:1305/1480 train_time:202243ms step_avg:156.17ms
step:1306/1480 train_time:202408ms step_avg:156.18ms
step:1307/1480 train_time:202570ms step_avg:156.18ms
step:1308/1480 train_time:202733ms step_avg:156.19ms
step:1309/1480 train_time:202898ms step_avg:156.20ms
step:1310/1480 train_time:203060ms step_avg:156.20ms
step:1311/1480 train_time:203221ms step_avg:156.20ms
step:1312/1480 train_time:203386ms step_avg:156.21ms
step:1313/1480 train_time:203548ms step_avg:156.21ms
step:1314/1480 train_time:203712ms step_avg:156.22ms
step:1315/1480 train_time:203877ms step_avg:156.23ms
step:1316/1480 train_time:204037ms step_avg:156.23ms
step:1317/1480 train_time:204199ms step_avg:156.23ms
step:1318/1480 train_time:204366ms step_avg:156.24ms
step:1319/1480 train_time:204532ms step_avg:156.25ms
step:1320/1480 train_time:204701ms step_avg:156.26ms
step:1321/1480 train_time:204863ms step_avg:156.27ms
step:1322/1480 train_time:205034ms step_avg:156.28ms
step:1323/1480 train_time:205199ms step_avg:156.28ms
step:1324/1480 train_time:205363ms step_avg:156.29ms
step:1325/1480 train_time:205534ms step_avg:156.30ms
step:1326/1480 train_time:205700ms step_avg:156.31ms
step:1327/1480 train_time:205862ms step_avg:156.31ms
step:1328/1480 train_time:206025ms step_avg:156.32ms
step:1329/1480 train_time:206214ms step_avg:156.34ms
step:1330/1480 train_time:206374ms step_avg:156.34ms
step:1331/1480 train_time:206537ms step_avg:156.35ms
step:1332/1480 train_time:206701ms step_avg:156.35ms
step:1333/1480 train_time:206866ms step_avg:156.36ms
step:1334/1480 train_time:207028ms step_avg:156.37ms
step:1335/1480 train_time:207188ms step_avg:156.37ms
step:1336/1480 train_time:207358ms step_avg:156.38ms
step:1337/1480 train_time:207524ms step_avg:156.39ms
step:1338/1480 train_time:207687ms step_avg:156.39ms
step:1339/1480 train_time:207852ms step_avg:156.40ms
step:1340/1480 train_time:208016ms step_avg:156.40ms
step:1341/1480 train_time:208179ms step_avg:156.41ms
step:1342/1480 train_time:208343ms step_avg:156.41ms
step:1343/1480 train_time:208504ms step_avg:156.42ms
step:1344/1480 train_time:208666ms step_avg:156.42ms
step:1345/1480 train_time:208836ms step_avg:156.43ms
step:1346/1480 train_time:208998ms step_avg:156.44ms
step:1347/1480 train_time:209160ms step_avg:156.44ms
step:1348/1480 train_time:209323ms step_avg:156.44ms
step:1349/1480 train_time:209485ms step_avg:156.45ms
step:1350/1480 train_time:209651ms step_avg:156.46ms
step:1351/1480 train_time:209813ms step_avg:156.46ms
step:1352/1480 train_time:209977ms step_avg:156.47ms
step:1353/1480 train_time:210142ms step_avg:156.47ms
step:1354/1480 train_time:210305ms step_avg:156.48ms
step:1355/1480 train_time:210468ms step_avg:156.48ms
step:1356/1480 train_time:210633ms step_avg:156.49ms
step:1357/1480 train_time:210797ms step_avg:156.49ms
step:1358/1480 train_time:210960ms step_avg:156.50ms
step:1359/1480 train_time:211124ms step_avg:156.50ms
step:1360/1480 train_time:211291ms step_avg:156.51ms
step:1361/1480 train_time:211459ms step_avg:156.52ms
step:1362/1480 train_time:211624ms step_avg:156.53ms
step:1363/1480 train_time:211792ms step_avg:156.53ms
step:1364/1480 train_time:211955ms step_avg:156.54ms
step:1365/1480 train_time:212115ms step_avg:156.54ms
step:1366/1480 train_time:212280ms step_avg:156.55ms
step:1367/1480 train_time:212443ms step_avg:156.55ms
step:1368/1480 train_time:212607ms step_avg:156.56ms
step:1369/1480 train_time:212778ms step_avg:156.57ms
step:1370/1480 train_time:212943ms step_avg:156.58ms
step:1371/1480 train_time:213106ms step_avg:156.58ms
step:1372/1480 train_time:213276ms step_avg:156.59ms
step:1373/1480 train_time:213437ms step_avg:156.59ms
step:1374/1480 train_time:213603ms step_avg:156.60ms
step:1375/1480 train_time:213765ms step_avg:156.60ms
step:1375/1480 val_loss:3.3020 train_time:213839ms step_avg:156.66ms
step:1376/1480 train_time:213932ms step_avg:156.61ms
step:1377/1480 train_time:214096ms step_avg:156.62ms
step:1378/1480 train_time:214258ms step_avg:156.62ms
step:1379/1480 train_time:214422ms step_avg:156.63ms
step:1380/1480 train_time:214586ms step_avg:156.63ms
step:1381/1480 train_time:214755ms step_avg:156.64ms
step:1382/1480 train_time:214918ms step_avg:156.65ms
step:1383/1480 train_time:215081ms step_avg:156.65ms
step:1384/1480 train_time:215248ms step_avg:156.66ms
step:1385/1480 train_time:215409ms step_avg:156.66ms
step:1386/1480 train_time:215571ms step_avg:156.66ms
step:1387/1480 train_time:215735ms step_avg:156.67ms
step:1388/1480 train_time:215896ms step_avg:156.67ms
step:1389/1480 train_time:216062ms step_avg:156.68ms
step:1390/1480 train_time:216224ms step_avg:156.68ms
step:1391/1480 train_time:216386ms step_avg:156.69ms
step:1392/1480 train_time:216549ms step_avg:156.69ms
step:1393/1480 train_time:216713ms step_avg:156.70ms
step:1394/1480 train_time:216875ms step_avg:156.70ms
step:1395/1480 train_time:217038ms step_avg:156.71ms
step:1396/1480 train_time:217200ms step_avg:156.71ms
step:1397/1480 train_time:217360ms step_avg:156.71ms
step:1398/1480 train_time:217521ms step_avg:156.72ms
step:1399/1480 train_time:217683ms step_avg:156.72ms
step:1400/1480 train_time:217853ms step_avg:156.73ms
step:1401/1480 train_time:218014ms step_avg:156.73ms
step:1402/1480 train_time:218175ms step_avg:156.73ms
step:1403/1480 train_time:218341ms step_avg:156.74ms
step:1404/1480 train_time:218503ms step_avg:156.75ms
step:1405/1480 train_time:218670ms step_avg:156.75ms
step:1406/1480 train_time:218836ms step_avg:156.76ms
step:1407/1480 train_time:218997ms step_avg:156.76ms
step:1408/1480 train_time:219158ms step_avg:156.77ms
step:1409/1480 train_time:219332ms step_avg:156.78ms
step:1410/1480 train_time:219494ms step_avg:156.78ms
step:1411/1480 train_time:219654ms step_avg:156.78ms
step:1412/1480 train_time:219815ms step_avg:156.79ms
step:1413/1480 train_time:219977ms step_avg:156.79ms
step:1414/1480 train_time:220142ms step_avg:156.80ms
step:1415/1480 train_time:220308ms step_avg:156.80ms
step:1416/1480 train_time:220480ms step_avg:156.81ms
step:1417/1480 train_time:220645ms step_avg:156.82ms
step:1418/1480 train_time:220809ms step_avg:156.82ms
step:1419/1480 train_time:220974ms step_avg:156.83ms
step:1420/1480 train_time:221139ms step_avg:156.84ms
step:1421/1480 train_time:221304ms step_avg:156.84ms
step:1422/1480 train_time:221468ms step_avg:156.85ms
step:1423/1480 train_time:221631ms step_avg:156.85ms
step:1424/1480 train_time:221797ms step_avg:156.86ms
step:1425/1480 train_time:221967ms step_avg:156.87ms
step:1426/1480 train_time:222132ms step_avg:156.87ms
step:1427/1480 train_time:222296ms step_avg:156.88ms
step:1428/1480 train_time:222458ms step_avg:156.88ms
step:1429/1480 train_time:222620ms step_avg:156.88ms
step:1430/1480 train_time:222784ms step_avg:156.89ms
step:1431/1480 train_time:222951ms step_avg:156.90ms
step:1432/1480 train_time:223118ms step_avg:156.90ms
step:1433/1480 train_time:223288ms step_avg:156.91ms
step:1434/1480 train_time:223457ms step_avg:156.92ms
step:1435/1480 train_time:223621ms step_avg:156.93ms
step:1436/1480 train_time:223787ms step_avg:156.93ms
step:1437/1480 train_time:223949ms step_avg:156.94ms
step:1438/1480 train_time:224111ms step_avg:156.94ms
step:1439/1480 train_time:224277ms step_avg:156.95ms
step:1440/1480 train_time:224440ms step_avg:156.95ms
step:1441/1480 train_time:224605ms step_avg:156.96ms
step:1442/1480 train_time:224772ms step_avg:156.96ms
step:1443/1480 train_time:224943ms step_avg:156.97ms
step:1444/1480 train_time:225107ms step_avg:156.98ms
step:1445/1480 train_time:225270ms step_avg:156.98ms
step:1446/1480 train_time:225435ms step_avg:156.99ms
step:1447/1480 train_time:225604ms step_avg:157.00ms
step:1448/1480 train_time:225767ms step_avg:157.00ms
step:1449/1480 train_time:225932ms step_avg:157.01ms
step:1450/1480 train_time:226097ms step_avg:157.01ms
step:1451/1480 train_time:226260ms step_avg:157.02ms
step:1452/1480 train_time:226425ms step_avg:157.02ms
step:1453/1480 train_time:226588ms step_avg:157.03ms
step:1454/1480 train_time:226751ms step_avg:157.03ms
step:1455/1480 train_time:226919ms step_avg:157.04ms
step:1456/1480 train_time:227080ms step_avg:157.04ms
step:1457/1480 train_time:227242ms step_avg:157.04ms
step:1458/1480 train_time:227407ms step_avg:157.05ms
step:1459/1480 train_time:227572ms step_avg:157.05ms
step:1460/1480 train_time:227736ms step_avg:157.06ms
step:1461/1480 train_time:227899ms step_avg:157.06ms
step:1462/1480 train_time:228064ms step_avg:157.07ms
step:1463/1480 train_time:228230ms step_avg:157.07ms
step:1464/1480 train_time:228396ms step_avg:157.08ms
step:1465/1480 train_time:228559ms step_avg:157.09ms
step:1466/1480 train_time:228722ms step_avg:157.09ms
step:1467/1480 train_time:228889ms step_avg:157.10ms
step:1468/1480 train_time:229053ms step_avg:157.10ms
step:1469/1480 train_time:229215ms step_avg:157.10ms
step:1470/1480 train_time:229383ms step_avg:157.11ms
step:1471/1480 train_time:229556ms step_avg:157.12ms
step:1472/1480 train_time:229727ms step_avg:157.13ms
step:1473/1480 train_time:229891ms step_avg:157.14ms
step:1474/1480 train_time:230058ms step_avg:157.14ms
step:1475/1480 train_time:230227ms step_avg:157.15ms
step:1476/1480 train_time:230391ms step_avg:157.16ms
step:1477/1480 train_time:230559ms step_avg:157.16ms
step:1478/1480 train_time:230730ms step_avg:157.17ms
step:1479/1480 train_time:230895ms step_avg:157.18ms
step:1480/1480 train_time:231057ms step_avg:157.18ms
step:1480/1480 val_loss:3.2831 train_time:231133ms step_avg:157.23ms
peak memory consumption: 34239 MiB