0aa83756-53f0-4268-9721-db6d5985bc42.txt

import os
import sys
with open(sys.argv[0]) as f:
    code = f.read() # read the code of this file ASAP, for logging
import uuid
import time
import contextlib
from dataclasses import dataclass
from pathlib import Path

import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.nn.attention.flex_attention import BlockMask, flex_attention #KoszarskyB

# -----------------------------------------------------------------------------
# Muon optimizer

@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
    """
    Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
    quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
    of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
    zero even beyond the point where the iteration no longer converges all the way to one everywhere
    on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
    where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
    performance at all relative to UV^T, where USV^T = G is the SVD.
    """
    assert len(G.shape) == 2
    a, b, c = (3.4445, -4.7750,  2.0315)
    X = G.bfloat16()
    X /= (X.norm() + eps) # ensure top singular value <= 1
    if G.size(0) > G.size(1):
        X = X.T
    for _ in range(steps):
        A = X @ X.T
        B = b * A + c * A @ A # adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
        X = a * X + B @ X
    if G.size(0) > G.size(1):
        X = X.T
    return X

class Muon(torch.optim.Optimizer):
    """
    Muon - MomentUm Orthogonalized by Newton-schulz

    Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
    processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
    matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
    the advantage that it can be stably run in bfloat16 on the GPU.

    Some warnings:
    - This optimizer assumes that all parameters passed in are 2D.
    - It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
    parameters; those should all be optimized by a standard method (e.g., AdamW).
    - To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
    - We believe it is unlikely to work well for training with small batch size.
    - We believe it may not work well for finetuning pretrained models, but we haven't tested this.
    - We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).

    Arguments:
        lr: The learning rate used by the internal SGD.
        momentum: The momentum used by the internal SGD.
        nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
        ns_steps: The number of Newton-Schulz iteration steps to use.
    """
    def __init__(self, params, lr=0.02, momentum=0.95, nesterov=True, ns_steps=5):
        self.world_size = int(os.environ['WORLD_SIZE'])
        self.rank = int(os.environ['RANK'])
        defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, ns_steps=ns_steps)
        params = list(params)
        assert all(isinstance(p, torch.Tensor) for p in params)
        sizes = {p.numel() for p in params}
        param_groups = [
            {
                'params': [p for p in params if p.numel() == size],
                'update_buffer': [
                    torch.empty(size, device='cuda', dtype=torch.bfloat16)
                    for _ in range(self.world_size)
                ],
            }
            for size in sizes
        ]
        super().__init__(param_groups, defaults)

    def step(self):

        for group in self.param_groups:

            lr = group['lr']
            momentum = group['momentum']
            nesterov = group['nesterov']
            ns_steps = group['ns_steps']
            update_buffers = group['update_buffer']
            # generate weight updates in distributed fashion
            params = group['params']
            assert len(params) % self.world_size == 0
            handle = None
            params_world = None
            def update_prev():
                if params_world is None:
                    return
                assert handle is not None
                handle.wait()
                for p_world, g_world in zip(params_world, update_buffers):
                    p_world.data.add_(
                        g_world.view_as(p_world),
                        alpha=-lr * max(1, p_world.size(0) / p_world.size(1)) ** 0.5,
                    )
            for base_i in range(len(params))[::self.world_size]:
                p = params[base_i + self.rank]
                g = p.grad
                assert g is not None
                state = self.state[p]
                if 'momentum_buffer' not in state:
                    state['momentum_buffer'] = torch.zeros_like(g)
                buf = state['momentum_buffer']
                buf.lerp_(g, 1 - momentum)
                g = g.lerp_(buf, momentum) if nesterov else buf
                g = zeropower_via_newtonschulz5(g, steps=ns_steps).flatten()
                update_prev()
                handle = dist.all_gather(update_buffers, g, async_op=True)
                params_world = params[base_i : base_i + self.world_size]
            update_prev()

# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model

def norm(x):
    return F.rms_norm(x, (x.size(-1),))

class CastedLinear(nn.Linear):

    def __init__(self, in_features, out_features):
        super().__init__(in_features, out_features, bias=False)

    def forward(self, x):
        return F.linear(x, self.weight.to(x.dtype))

class Rotary(torch.nn.Module):

    def __init__(self, dim, base=10000):
        super().__init__()
        self.register_buffer('inv_freq', (1 / base) ** (torch.arange(0, dim, 2) / dim))
        self.seq_len_cached = None
        self.cos_cached = None
        self.sin_cached = None

    def forward(self, x):
        seq_len = x.shape[1]
        if seq_len != self.seq_len_cached:
            t = torch.arange(seq_len, device=x.device)
            freqs = torch.outer(t, self.inv_freq)
            self.seq_len_cached = seq_len
            self.cos_cached = freqs.cos()
            self.sin_cached = freqs.sin()
        cos, sin = self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
        # apply_rotary_emb(x, cos, sin)
        x1, x2 = x.chunk(2, dim=3)
        y1 = x1 * cos + x2 * sin
        y2 = x1 * (-sin) + x2 * cos
        return torch.cat((y1, y2), 3).type_as(x)

class CausalSelfAttention(nn.Module):

    def __init__(self, dim, num_heads):
        super().__init__()
        assert dim % num_heads == 0
        self.num_heads = num_heads
        self.c_q = CastedLinear(dim, dim)
        self.c_k = CastedLinear(dim, dim)
        self.c_v = CastedLinear(dim, dim)
        self.lambdas = nn.Parameter(torch.tensor([0.5, 0.5]))
        self.rotary = Rotary(dim // num_heads) # dim // num_heads = head_dim
        self.c_proj = CastedLinear(dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x, vi, block_mask):
        B, T = x.size(0), x.size(1) # batch size, sequence length
        assert B == 1, "Must use batch size = 1 for FlexAttention"
        q = self.c_q(x).view(B, T, self.num_heads, -1)
        k = self.c_k(x).view(B, T, self.num_heads, -1)
        v = self.c_v(x).view(B, T, self.num_heads, -1)
        v = self.lambdas[0] * v + self.lambdas[1] * vi.view_as(v) # @KoszarskyB & @Grad62304977
        q, k = norm(q), norm(k) # QK norm @Grad62304977
        q, k = self.rotary(q), self.rotary(k)
        y = flex_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), block_mask=block_mask, enable_gqa=True)
        y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
        y = self.c_proj(y)
        return y

class MLP(nn.Module):

    def __init__(self, dim):
        super().__init__()
        self.c_fc   = CastedLinear(dim, 4 * dim)
        self.c_proj = CastedLinear(4 * dim, dim)
        self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977

    def forward(self, x):
        x = self.c_fc(x)
        x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
        x = self.c_proj(x)
        return x

class Block(nn.Module):

    def __init__(self, config):
        super().__init__()
        self.attn = CausalSelfAttention(config.model_dim, config.num_heads)
        self.mlp = MLP(config.model_dim)
        self.lambdas = nn.Parameter(torch.tensor([1., 0.]))

    def forward(self, x, vi, x0, block_mask):
        x = self.lambdas[0] * x + self.lambdas[1] * x0
        x = x + self.attn(norm(x), vi, block_mask)
        x = x + self.mlp(norm(x))
        return x

class ValueEmbedding(nn.Module):
    def __init__(self, config: "GPTConfig"):
        super().__init__()
        self.__setattr__
        self.embed = nn.ModuleList([
            nn.Embedding(config.vocab_size, config.model_dim)
            for _ in range(6)
        ])

    def forward(self, inputs) -> "list[torch.Tensor]":
        ve = [emb(inputs) for emb in self.embed]
        ve += reversed(ve)
        return ve


# -----------------------------------------------------------------------------
# The main GPT-2 model

@dataclass
class GPTConfig:
    vocab_size : int = 50304
    num_layers : int = 12
    num_heads : int = 6 # head dim 128 suggested by @Grad62304977
    model_dim : int = 768

class GPT(nn.Module):

    def __init__(self, config: GPTConfig):
        super().__init__()
        self.num_layers = config.num_layers

        # U-net design by @brendanh0gan
        self.num_encoder_layers = config.num_layers // 2 # Half of the layers for encoder
        self.num_decoder_layers = config.num_layers - self.num_encoder_layers # Remaining for decoder
        # Add learnable skip connection weights for decoder layers
        self.skip_weights = nn.Parameter(torch.ones(self.num_decoder_layers))

        self.embed = nn.Embedding(config.vocab_size, config.model_dim)
        self.blocks = nn.ModuleList([Block(config) for _ in range(config.num_layers)])
        # token value embeddings by @KoszarskyB - inspired by @Grad62304977's value residual learning
        # U-net structure on token value embeddings by @leloykun
        self.value_embeds = ValueEmbedding(config)
        self.lm_head = CastedLinear(config.model_dim, config.vocab_size)
        self.lm_head.weight.data.zero_() # @Grad62304977

    def forward(
        self,
        inputs: torch.Tensor,
        targets: torch.Tensor,
        sliding_window_num_blocks: torch.Tensor,
    ):
        BLOCK_SIZE = 128
        assert inputs.ndim == 1
        docs = (inputs == 50256).cumsum(0)
        docs_low = docs.view(-1, BLOCK_SIZE)[:, 0].contiguous()
        docs_high = docs.view(-1, BLOCK_SIZE)[:, -1].contiguous()

        def document_causal(b, h, q_idx, kv_idx):
            causal_mask = q_idx >= kv_idx
            document_mask = docs[q_idx] == docs[kv_idx]
            return causal_mask & document_mask

        def dense_to_ordered(dense_mask: torch.Tensor):
            num_blocks = dense_mask.sum(dim=-1, dtype=torch.int32)
            indices = dense_mask.argsort(dim=-1, descending=True, stable=True).to(torch.int32)
            return num_blocks[None, None].contiguous(), indices[None, None].contiguous()

        def create_doc_swc_block_mask(sliding_window_num_blocks: torch.Tensor):
            kv_idx = block_idx = torch.arange(512, dtype=torch.int32, device="cuda")
            q_idx = block_idx[:, None]
            causal_bm = q_idx >= kv_idx
            causal_full_bm = q_idx > kv_idx
            window_bm = q_idx - kv_idx < sliding_window_num_blocks
            window_full_bm = window_bm
            # document_bm = (docs_low[q_idx] <= docs_high[kv_idx]) & (docs_low[kv_idx] <= docs_high[q_idx])
            document_bm = (docs_low[:, None] <= docs_high) & (docs_low <= docs_high[:, None])
            document_full_bm = (docs_low[:, None] == docs_high) & (docs_low == docs_high[:, None])
            nonzero_bm = causal_bm & window_bm & document_bm
            full_bm  = causal_full_bm & window_full_bm & document_full_bm
            kv_num_blocks, kv_indices = dense_to_ordered(nonzero_bm ^ full_bm)
            full_kv_num_blocks, full_kv_indices = dense_to_ordered(full_bm)
            return BlockMask.from_kv_blocks(
                kv_num_blocks,
                kv_indices,
                full_kv_num_blocks,
                full_kv_indices,
                BLOCK_SIZE=BLOCK_SIZE,
                mask_mod=document_causal,
            )

        block_mask = create_doc_swc_block_mask(sliding_window_num_blocks)

        # forward the GPT model itself
        x = self.embed(inputs[None]) # token embeddings of shape (b, t, model_dim)
        x = norm(x) # @Grad62304977
        x0 = x
        ve = self.value_embeds(inputs)
        ve_enc, ve_dec = ve[:self.num_encoder_layers], ve[self.num_encoder_layers:]

        # Store outputs for U-Net skip connections
        skip_connections = []
        # Encoder pass - process only the first half of the blocks
        for i in range(self.num_encoder_layers):
            x = self.blocks[i](x, ve_enc[i], x0, block_mask)
            skip_connections.append(x)
        # Decoder pass - process the remaining blocks with weighted skip connections
        for i in range(self.num_decoder_layers):
            x = x + self.skip_weights[i] * skip_connections.pop()
            # U-net structure on token value embeddings by @leloykun
            x = self.blocks[self.num_encoder_layers + i](x, ve_dec[i], x0, block_mask)

        x = norm(x)
        logits = self.lm_head(x)
        logits = 30 * torch.tanh(logits / 30) # @Grad62304977
        logits = logits.float()
        loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
        return loss

# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader

def _peek_data_shard(file: Path):
    # only reads the header, returns header data
    # header is 256 int32
    header = torch.from_file(f"{file}", False, 256, dtype=torch.int32)
    assert header[0] == 20240520, "magic number mismatch in the data .bin file"
    assert header[1] == 1, "unsupported version"
    return int(header[2]) # number of tokens (claimed)

def _load_data_shard(path: Path, num_tokens):
    with path.open("rb", buffering=0) as f:
        tokens = torch.empty(num_tokens, dtype=torch.uint16, pin_memory=True)
        f.seek(256 * 4)
        nbytes = f.readinto(tokens.numpy())
        assert nbytes == 2 * num_tokens, "number of tokens read does not match header?"
    return tokens

class DistributedDataLoader:
    def __init__(self, filename_pattern, seq_len, process_rank, num_processes):
        self.process_rank = process_rank
        self.num_processes = num_processes
        self.seq_len = seq_len

        # glob files that match the pattern
        self.files = sorted(Path.cwd().glob(filename_pattern))
        assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"

        # load and validate all data shards, count number of tokens in total
        self.files_num_tokens = [_peek_data_shard(file) for file in self.files]
        assert min(self.files_num_tokens) >= num_processes * seq_len + 1
        self.total_num_tokens = sum(self.files_num_tokens)

        self.reset()

    def reset(self):
        self.current_shard = -1
        self.advance()

    def advance(self): # advance to next data shard
        self.current_shard = (self.current_shard + 1) % len(self.files)
        self.current_position = self.process_rank * self.seq_len
        self.tokens = _load_data_shard(self.files[self.current_shard], self.files_num_tokens[self.current_shard])

    def next_batch(self):
        batch_size = self.seq_len * self.num_processes
        buf = self.tokens[self.current_position:self.current_position+self.seq_len+1]
        # host side async is sufficient;
        # no performance improvement was observed when introducing a separate stream.
        inputs = buf[:-1].to(device="cuda", dtype=torch.int32, non_blocking=True) # inputs
        targets = buf[1:].to(device="cuda", dtype=torch.int64, non_blocking=True) # targets
        # advance current position and load next shard if necessary
        self.current_position += batch_size
        if self.current_position + batch_size + 1 >= len(self.tokens):
            self.advance()
        return inputs, targets

# -----------------------------------------------------------------------------
# int main

@dataclass
class Hyperparameters:
    # data hyperparams
    input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
    input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
    # optimization hyperparams
    batch_size : int = 8 # batch size, in sequences, across all devices
    sequence_length : int = 64*1024 # sequence length, in tokens
    num_iterations : int = 1480 # number of iterations to run
    warmup_iters : int = 0
    cooldown_iters : int = 600 # number of iterations of linear warmup/cooldown for triangular or trapezoidal schedule
    weight_decay : float = 0
    # evaluation and logging hyperparams
    val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
    val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
    save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()

# set up DDP (distributed data parallel). torchrun sets this env variable
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
assert torch.cuda.is_available()
device = torch.device(f"cuda:{ddp_local_rank}")
torch.cuda.set_device(device)
print(f"using device: {device}")
dist.init_process_group(backend='nccl', device_id=device)
dist.barrier()
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.

# begin logging
logfile = None
if master_process:
    run_id = uuid.uuid4()
    logdir = Path("logs") / f"{run_id}"
    logdir.mkdir(exist_ok=True)
    logfile = Path("logs") / f"{run_id}.txt"
    print(logfile.stem)
    # create the log file
    with logfile.open("w") as f:
        # begin the log by printing this file (the Python code)
        print(code, file=f)
        print("=" * 100, file=f)
def print0(s, logonly=False):
    if master_process:
        with logfile.open("a") as f:
            if not logonly:
                print(s)
            print(s, file=f)
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
print0(f"Running python {sys.version}")
print0(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
print0(f'{result.stdout}', logonly=True)
print0('='*100, logonly=True)

# calculate the number of steps to take in the val loop.
assert args.val_tokens % (args.sequence_length * ddp_world_size) == 0
val_steps = args.val_tokens // (args.sequence_length * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (ddp_world_size) == 0
train_accumulation_steps = args.batch_size // ddp_world_size

# load tokens
train_loader = DistributedDataLoader(args.input_bin, args.sequence_length, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, args.sequence_length, ddp_rank, ddp_world_size)
print0(f"Training DataLoader: total number of tokens: {train_loader.total_num_tokens} across {len(train_loader.files)} files")
print0(f"Validation DataLoader: total number of tokens: {val_loader.total_num_tokens} across {len(val_loader.files)} files")
print0('='*100, logonly=True)
inputs_train, targets_train = train_loader.next_batch()

# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, num_layers=12, num_heads=6, model_dim=768))
model = model.cuda().bfloat16()
for m in model.modules():
    if isinstance(m, CastedLinear):
        m.float()
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank], broadcast_buffers=False, gradient_as_bucket_view=True)
raw_model = model.module # always contains the "raw" unwrapped model

# init the optimizer(s)
embed_params = [*raw_model.embed.parameters(), *raw_model.value_embeds.parameters()]
optimizer1 = torch.optim.Adam(embed_params, lr=0.6, betas=(0.8, 0.95), fused=True)
optimizer2 = torch.optim.Adam([raw_model.lm_head.weight], lr=0.008, betas=(0.8, 0.95), fused=True)
params = list(raw_model.blocks.parameters())
matrix_params = [p for p in params if p.ndim == 2]
scalar_params = [p for p in params if p.ndim < 2] + [raw_model.skip_weights]
optimizer3 = Muon(matrix_params, lr=0.05, momentum=0.95)
optimizer4 = torch.optim.Adam(scalar_params, lr=0.04, betas=(0.8, 0.95), fused=True)
optimizers = [optimizer1, optimizer2, optimizer3, optimizer4]
# learning rate decay scheduler (linear warmup and cooldown)
def get_lr(it):
    assert it <= args.num_iterations
    # 1) linear warmup for warmup_iters steps
    if it < args.warmup_iters:
        return (it+1) / args.warmup_iters
    # 2) constant lr for a while
    elif it < args.num_iterations - args.cooldown_iters:
        return 1.0
    # 3) linear cooldown
    else:
        decay_ratio = (args.num_iterations - it) / args.cooldown_iters
        return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]

sliding_window_num_blocks = torch.tensor(1, dtype=torch.int32, device="cuda")
sw_num_blocks_prev = 1
# Start training loop
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.perf_counter()
# begin training
for step in range(args.num_iterations + 1):
    last_step = (step == args.num_iterations)
    # This effectively ignores timing first 10 steps, which are slower for weird reasons.
    # Alternately, and slightly more correctly in terms of benchmarking, we could do 10
    # steps with dummy data first, and then re-initialize the model and reset the loader.
    if step == 10:
        training_time_ms = 0
        t0 = time.perf_counter()
    timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val

    # Linearly increase the sliding window size over training in chunks of 64 from 64 -> 1792. By @fernbear.bsky.social
    frac_done = step / args.num_iterations # training progress
    sw_num_blocks = int(((1 - frac_done) * 64 + frac_done * 1792 + 64) // 128)
    if sw_num_blocks != sw_num_blocks_prev:
        sliding_window_num_blocks.copy_(sw_num_blocks, non_blocking=True)
        sw_num_blocks_prev = sw_num_blocks

    # once in a while evaluate the validation dataset
    if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # run validation batches
        model.eval()
        val_loader.reset()
        val_loss = 0.0
        for _ in range(val_steps):
            with torch.no_grad():
                inputs_val, targets_val = val_loader.next_batch()
                val_loss += model(inputs_val, targets_val, sliding_window_num_blocks)
        dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
        val_loss /= val_steps
        # log val loss to console and to logfile
        print0(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
        # stop the clock
        torch.cuda.synchronize()
        training_time_ms += 1000 * (time.perf_counter() - t0)
        # save the state of the training process
        log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
        torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
        # start the clock again
        torch.cuda.synchronize()
        t0 = time.perf_counter()

    # bit confusing: we want to make sure to eval on 0th iteration
    # but also after the very last iteration. so we loop for step <= num_iterations
    # instead of just < num_iterations (one extra due to <=), only to do
    # the validation/sampling one last time, and then we break right here as we're done.
    if last_step:
        break

    # --------------- TRAINING SECTION BEGIN -----------------
    model.train()
    for i in range(1, train_accumulation_steps + 1):
        with contextlib.ExitStack() as stack:
            if i < train_accumulation_steps: # there's no need to sync gradients every accumulation step
                stack.enter_context(model.no_sync())
            if step >= 5:
                stack.enter_context(torch.compiler.set_stance(skip_guard_eval_unsafe=True))
            model(inputs_train, targets_train, sliding_window_num_blocks).backward()
            inputs_train, targets_train = train_loader.next_batch()
    if train_accumulation_steps != 1:
        for p in model.parameters():
            p.grad /= train_accumulation_steps
    # momentum warmup for Muon
    frac = min(step/300, 1)
    for group in optimizer3.param_groups:
        group['momentum'] = (1 - frac) * 0.85 + frac * 0.95
    # step the optimizers and schedulers
    for opt, sched in zip(optimizers, schedulers):
        opt.step()
        sched.step()
    # null the gradients
    model.zero_grad(set_to_none=True)
    # --------------- TRAINING SECTION END -------------------
    # everything that follows now is just diagnostics, prints, logging, etc.
    approx_time = training_time_ms + 1000 * (time.perf_counter() - t0)
    print0(f"step:{step+1}/{args.num_iterations} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")

print0(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")

# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()

====================================================================================================
Running python 3.10.12 (main, Sep 11 2024, 15:47:36) [GCC 11.4.0]
Running pytorch 2.6.0.dev20241203+cu124 compiled for CUDA 12.4
nvidia-smi:
Wed Dec 11 09:52:03 2024       
+---------------------------------------------------------------------------------------+
| NVIDIA-SMI 535.183.06             Driver Version: 535.183.06   CUDA Version: 12.2     |
|-----------------------------------------+----------------------+----------------------+
| GPU  Name                 Persistence-M | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp   Perf          Pwr:Usage/Cap |         Memory-Usage | GPU-Util  Compute M. |
|                                         |                      |               MIG M. |
|=========================================+======================+======================|
|   0  NVIDIA H100 80GB HBM3          On  | 00000000:19:00.0 Off |                    0 |
| N/A   38C    P0             126W / 700W |   7084MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   1  NVIDIA H100 80GB HBM3          On  | 00000000:3B:00.0 Off |                    0 |
| N/A   30C    P0             116W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   2  NVIDIA H100 80GB HBM3          On  | 00000000:4C:00.0 Off |                    0 |
| N/A   29C    P0             112W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   3  NVIDIA H100 80GB HBM3          On  | 00000000:5D:00.0 Off |                    0 |
| N/A   37C    P0             114W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   4  NVIDIA H100 80GB HBM3          On  | 00000000:9B:00.0 Off |                    0 |
| N/A   38C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   5  NVIDIA H100 80GB HBM3          On  | 00000000:BB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   6  NVIDIA H100 80GB HBM3          On  | 00000000:CB:00.0 Off |                    0 |
| N/A   36C    P0             120W / 700W |   3451MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
|   7  NVIDIA H100 80GB HBM3          On  | 00000000:DB:00.0 Off |                    0 |
| N/A   30C    P0             118W / 700W |   3211MiB / 81559MiB |      0%      Default |
|                                         |                      |             Disabled |
+-----------------------------------------+----------------------+----------------------+
                                                                                         
+---------------------------------------------------------------------------------------+
| Processes:                                                                            |
|  GPU   GI   CI        PID   Type   Process name                            GPU Memory |
|        ID   ID                                                             Usage      |
|=======================================================================================|
+---------------------------------------------------------------------------------------+

====================================================================================================
Training DataLoader: total number of tokens: 1000000000 across 10 files
Validation DataLoader: total number of tokens: 100000000 across 1 files
====================================================================================================
step:0/1480 val_loss:10.8258 train_time:0ms step_avg:nanms
step:1/1480 train_time:28926ms step_avg:nanms
step:2/1480 train_time:29039ms step_avg:nanms
step:3/1480 train_time:29151ms step_avg:nanms
step:4/1480 train_time:29291ms step_avg:nanms
step:5/1480 train_time:29433ms step_avg:nanms
step:6/1480 train_time:29574ms step_avg:nanms
step:7/1480 train_time:29715ms step_avg:nanms
step:8/1480 train_time:29857ms step_avg:nanms
step:9/1480 train_time:30000ms step_avg:nanms
step:10/1480 train_time:30143ms step_avg:nanms
step:11/1480 train_time:142ms step_avg:nanms
step:12/1480 train_time:282ms step_avg:nanms
step:13/1480 train_time:425ms step_avg:141.61ms
step:14/1480 train_time:567ms step_avg:141.75ms
step:15/1480 train_time:709ms step_avg:141.86ms
step:16/1480 train_time:854ms step_avg:142.31ms
step:17/1480 train_time:997ms step_avg:142.48ms
step:18/1480 train_time:1142ms step_avg:142.81ms
step:19/1480 train_time:1283ms step_avg:142.61ms
step:20/1480 train_time:1426ms step_avg:142.59ms
step:21/1480 train_time:1568ms step_avg:142.54ms
step:22/1480 train_time:1709ms step_avg:142.43ms
step:23/1480 train_time:1854ms step_avg:142.64ms
step:24/1480 train_time:1999ms step_avg:142.80ms
step:25/1480 train_time:2141ms step_avg:142.75ms
step:26/1480 train_time:2284ms step_avg:142.76ms
step:27/1480 train_time:2426ms step_avg:142.73ms
step:28/1480 train_time:2569ms step_avg:142.72ms
step:29/1480 train_time:2711ms step_avg:142.69ms
step:30/1480 train_time:2854ms step_avg:142.70ms
step:31/1480 train_time:2997ms step_avg:142.72ms
step:32/1480 train_time:3142ms step_avg:142.81ms
step:33/1480 train_time:3283ms step_avg:142.75ms
step:34/1480 train_time:3425ms step_avg:142.71ms
step:35/1480 train_time:3567ms step_avg:142.67ms
step:36/1480 train_time:3708ms step_avg:142.63ms
step:37/1480 train_time:3852ms step_avg:142.65ms
step:38/1480 train_time:3994ms step_avg:142.65ms
step:39/1480 train_time:4138ms step_avg:142.68ms
step:40/1480 train_time:4280ms step_avg:142.67ms
step:41/1480 train_time:4423ms step_avg:142.68ms
step:42/1480 train_time:4565ms step_avg:142.67ms
step:43/1480 train_time:4707ms step_avg:142.63ms
step:44/1480 train_time:4848ms step_avg:142.58ms
step:45/1480 train_time:4992ms step_avg:142.62ms
step:46/1480 train_time:5135ms step_avg:142.65ms
step:47/1480 train_time:5278ms step_avg:142.65ms
step:48/1480 train_time:5422ms step_avg:142.68ms
step:49/1480 train_time:5564ms step_avg:142.67ms
step:50/1480 train_time:5706ms step_avg:142.65ms
step:51/1480 train_time:5847ms step_avg:142.60ms
step:52/1480 train_time:5989ms step_avg:142.58ms
step:53/1480 train_time:6133ms step_avg:142.63ms
step:54/1480 train_time:6276ms step_avg:142.64ms
step:55/1480 train_time:6419ms step_avg:142.65ms
step:56/1480 train_time:6562ms step_avg:142.66ms
step:57/1480 train_time:6705ms step_avg:142.66ms
step:58/1480 train_time:6845ms step_avg:142.60ms
step:59/1480 train_time:6986ms step_avg:142.57ms
step:60/1480 train_time:7129ms step_avg:142.59ms
step:61/1480 train_time:7272ms step_avg:142.59ms
step:62/1480 train_time:7416ms step_avg:142.61ms
step:63/1480 train_time:7559ms step_avg:142.62ms
step:64/1480 train_time:7702ms step_avg:142.62ms
step:65/1480 train_time:7843ms step_avg:142.59ms
step:66/1480 train_time:7984ms step_avg:142.57ms
step:67/1480 train_time:8127ms step_avg:142.58ms
step:68/1480 train_time:8269ms step_avg:142.57ms
step:69/1480 train_time:8413ms step_avg:142.59ms
step:70/1480 train_time:8558ms step_avg:142.63ms
step:71/1480 train_time:8701ms step_avg:142.64ms
step:72/1480 train_time:8843ms step_avg:142.63ms
step:73/1480 train_time:8984ms step_avg:142.60ms
step:74/1480 train_time:9126ms step_avg:142.59ms
step:75/1480 train_time:9267ms step_avg:142.57ms
step:76/1480 train_time:9410ms step_avg:142.57ms
step:77/1480 train_time:9555ms step_avg:142.61ms
step:78/1480 train_time:9698ms step_avg:142.62ms
step:79/1480 train_time:9840ms step_avg:142.61ms
step:80/1480 train_time:10361ms step_avg:148.01ms
step:81/1480 train_time:10464ms step_avg:147.37ms
step:82/1480 train_time:10606ms step_avg:147.30ms
step:83/1480 train_time:10746ms step_avg:147.21ms
step:84/1480 train_time:10888ms step_avg:147.13ms
step:85/1480 train_time:11030ms step_avg:147.07ms
step:86/1480 train_time:11172ms step_avg:146.99ms
step:87/1480 train_time:11315ms step_avg:146.95ms
step:88/1480 train_time:11837ms step_avg:151.76ms
step:89/1480 train_time:11942ms step_avg:151.16ms
step:90/1480 train_time:12083ms step_avg:151.04ms
step:91/1480 train_time:12226ms step_avg:150.94ms
step:92/1480 train_time:12366ms step_avg:150.81ms
step:93/1480 train_time:12508ms step_avg:150.70ms
step:94/1480 train_time:12649ms step_avg:150.58ms
step:95/1480 train_time:12792ms step_avg:150.50ms
step:96/1480 train_time:13309ms step_avg:154.76ms
step:97/1480 train_time:13415ms step_avg:154.19ms
step:98/1480 train_time:13558ms step_avg:154.06ms
step:99/1480 train_time:13701ms step_avg:153.95ms
step:100/1480 train_time:13844ms step_avg:153.82ms
step:101/1480 train_time:13987ms step_avg:153.71ms
step:102/1480 train_time:14126ms step_avg:153.54ms
step:103/1480 train_time:14267ms step_avg:153.40ms
step:104/1480 train_time:14409ms step_avg:153.29ms
step:105/1480 train_time:14552ms step_avg:153.18ms
step:106/1480 train_time:14695ms step_avg:153.08ms
step:107/1480 train_time:14839ms step_avg:152.98ms
step:108/1480 train_time:14983ms step_avg:152.89ms
step:109/1480 train_time:15126ms step_avg:152.79ms
step:110/1480 train_time:15269ms step_avg:152.69ms
step:111/1480 train_time:15412ms step_avg:152.59ms
step:112/1480 train_time:15559ms step_avg:152.54ms
step:113/1480 train_time:15705ms step_avg:152.47ms
step:114/1480 train_time:15849ms step_avg:152.39ms
step:115/1480 train_time:15994ms step_avg:152.33ms
step:116/1480 train_time:16141ms step_avg:152.28ms
step:117/1480 train_time:16286ms step_avg:152.20ms
step:118/1480 train_time:16431ms step_avg:152.14ms
step:119/1480 train_time:16577ms step_avg:152.08ms
step:120/1480 train_time:16723ms step_avg:152.03ms
step:121/1480 train_time:16867ms step_avg:151.96ms
step:122/1480 train_time:17013ms step_avg:151.90ms
step:123/1480 train_time:17160ms step_avg:151.86ms
step:124/1480 train_time:17305ms step_avg:151.80ms
step:125/1480 train_time:17450ms step_avg:151.74ms
step:125/1480 val_loss:4.4157 train_time:17514ms step_avg:152.30ms
step:126/1480 train_time:17605ms step_avg:151.77ms
step:127/1480 train_time:17750ms step_avg:151.71ms
step:128/1480 train_time:17897ms step_avg:151.67ms
step:129/1480 train_time:18041ms step_avg:151.60ms
step:130/1480 train_time:18185ms step_avg:151.54ms
step:131/1480 train_time:18330ms step_avg:151.49ms
step:132/1480 train_time:18476ms step_avg:151.44ms
step:133/1480 train_time:18621ms step_avg:151.39ms
step:134/1480 train_time:18768ms step_avg:151.36ms
step:135/1480 train_time:18915ms step_avg:151.32ms
step:136/1480 train_time:19061ms step_avg:151.27ms
step:137/1480 train_time:19206ms step_avg:151.23ms
step:138/1480 train_time:19351ms step_avg:151.18ms
step:139/1480 train_time:19498ms step_avg:151.15ms
step:140/1480 train_time:19642ms step_avg:151.10ms
step:141/1480 train_time:19790ms step_avg:151.07ms
step:142/1480 train_time:19936ms step_avg:151.03ms
step:143/1480 train_time:20082ms step_avg:150.99ms
step:144/1480 train_time:20228ms step_avg:150.95ms
step:145/1480 train_time:20375ms step_avg:150.93ms
step:146/1480 train_time:20521ms step_avg:150.89ms
step:147/1480 train_time:20665ms step_avg:150.84ms
step:148/1480 train_time:20811ms step_avg:150.80ms
step:149/1480 train_time:20958ms step_avg:150.78ms
step:150/1480 train_time:21104ms step_avg:150.74ms
step:151/1480 train_time:21249ms step_avg:150.70ms
step:152/1480 train_time:21396ms step_avg:150.67ms
step:153/1480 train_time:21540ms step_avg:150.63ms
step:154/1480 train_time:21685ms step_avg:150.59ms
step:155/1480 train_time:21831ms step_avg:150.56ms
step:156/1480 train_time:21978ms step_avg:150.53ms
step:157/1480 train_time:22122ms step_avg:150.49ms
step:158/1480 train_time:22269ms step_avg:150.46ms
step:159/1480 train_time:22417ms step_avg:150.45ms
step:160/1480 train_time:22563ms step_avg:150.42ms
step:161/1480 train_time:22708ms step_avg:150.38ms
step:162/1480 train_time:22854ms step_avg:150.36ms
step:163/1480 train_time:22999ms step_avg:150.32ms
step:164/1480 train_time:23144ms step_avg:150.28ms
step:165/1480 train_time:23289ms step_avg:150.25ms
step:166/1480 train_time:23436ms step_avg:150.23ms
step:167/1480 train_time:23582ms step_avg:150.20ms
step:168/1480 train_time:23726ms step_avg:150.16ms
step:169/1480 train_time:23874ms step_avg:150.15ms
step:170/1480 train_time:24019ms step_avg:150.12ms
step:171/1480 train_time:24164ms step_avg:150.09ms
step:172/1480 train_time:24309ms step_avg:150.06ms
step:173/1480 train_time:24456ms step_avg:150.04ms
step:174/1480 train_time:24601ms step_avg:150.00ms
step:175/1480 train_time:24746ms step_avg:149.97ms
step:176/1480 train_time:24892ms step_avg:149.95ms
step:177/1480 train_time:25038ms step_avg:149.93ms
step:178/1480 train_time:25182ms step_avg:149.89ms
step:179/1480 train_time:25327ms step_avg:149.86ms
step:180/1480 train_time:25474ms step_avg:149.85ms
step:181/1480 train_time:25620ms step_avg:149.82ms
step:182/1480 train_time:25766ms step_avg:149.80ms
step:183/1480 train_time:25912ms step_avg:149.78ms
step:184/1480 train_time:26058ms step_avg:149.76ms
step:185/1480 train_time:26201ms step_avg:149.72ms
step:186/1480 train_time:26346ms step_avg:149.69ms
step:187/1480 train_time:26493ms step_avg:149.68ms
step:188/1480 train_time:26639ms step_avg:149.66ms
step:189/1480 train_time:26802ms step_avg:149.73ms
step:190/1480 train_time:26929ms step_avg:149.61ms
step:191/1480 train_time:27076ms step_avg:149.59ms
step:192/1480 train_time:27220ms step_avg:149.56ms
step:193/1480 train_time:27366ms step_avg:149.54ms
step:194/1480 train_time:27511ms step_avg:149.52ms
step:195/1480 train_time:27657ms step_avg:149.50ms
step:196/1480 train_time:27802ms step_avg:149.47ms
step:197/1480 train_time:27946ms step_avg:149.45ms
step:198/1480 train_time:28093ms step_avg:149.43ms
step:199/1480 train_time:28238ms step_avg:149.41ms
step:200/1480 train_time:28383ms step_avg:149.39ms
step:201/1480 train_time:28529ms step_avg:149.37ms
step:202/1480 train_time:28676ms step_avg:149.36ms
step:203/1480 train_time:28821ms step_avg:149.33ms
step:204/1480 train_time:28966ms step_avg:149.31ms
step:205/1480 train_time:29112ms step_avg:149.29ms
step:206/1480 train_time:29258ms step_avg:149.28ms
step:207/1480 train_time:29402ms step_avg:149.25ms
step:208/1480 train_time:29547ms step_avg:149.23ms
step:209/1480 train_time:29694ms step_avg:149.22ms
step:210/1480 train_time:29839ms step_avg:149.20ms
step:211/1480 train_time:29986ms step_avg:149.18ms
step:212/1480 train_time:30132ms step_avg:149.17ms
step:213/1480 train_time:30278ms step_avg:149.15ms
step:214/1480 train_time:30422ms step_avg:149.13ms
step:215/1480 train_time:30568ms step_avg:149.11ms
step:216/1480 train_time:30714ms step_avg:149.10ms
step:217/1480 train_time:30860ms step_avg:149.08ms
step:218/1480 train_time:31004ms step_avg:149.06ms
step:219/1480 train_time:31150ms step_avg:149.04ms
step:220/1480 train_time:31296ms step_avg:149.03ms
step:221/1480 train_time:31850ms step_avg:150.95ms
step:222/1480 train_time:31961ms step_avg:150.76ms
step:223/1480 train_time:32108ms step_avg:150.74ms
step:224/1480 train_time:32257ms step_avg:150.73ms
step:225/1480 train_time:32403ms step_avg:150.71ms
step:226/1480 train_time:32552ms step_avg:150.70ms
step:227/1480 train_time:32699ms step_avg:150.69ms
step:228/1480 train_time:32848ms step_avg:150.68ms
step:229/1480 train_time:32998ms step_avg:150.67ms
step:230/1480 train_time:33144ms step_avg:150.65ms
step:231/1480 train_time:33293ms step_avg:150.65ms
step:232/1480 train_time:33441ms step_avg:150.64ms
step:233/1480 train_time:33589ms step_avg:150.62ms
step:234/1480 train_time:33738ms step_avg:150.62ms
step:235/1480 train_time:33887ms step_avg:150.61ms
step:236/1480 train_time:34036ms step_avg:150.60ms
step:237/1480 train_time:34183ms step_avg:150.59ms
step:238/1480 train_time:34332ms step_avg:150.58ms
step:239/1480 train_time:34480ms step_avg:150.57ms
step:240/1480 train_time:34627ms step_avg:150.55ms
step:241/1480 train_time:34776ms step_avg:150.55ms
step:242/1480 train_time:34925ms step_avg:150.54ms
step:243/1480 train_time:35074ms step_avg:150.53ms
step:244/1480 train_time:35221ms step_avg:150.52ms
step:245/1480 train_time:35370ms step_avg:150.51ms
step:246/1480 train_time:35519ms step_avg:150.51ms
step:247/1480 train_time:35667ms step_avg:150.49ms
step:248/1480 train_time:35816ms step_avg:150.49ms
step:249/1480 train_time:35964ms step_avg:150.48ms
step:250/1480 train_time:36112ms step_avg:150.47ms
step:250/1480 val_loss:3.9874 train_time:36179ms step_avg:150.74ms
step:251/1480 train_time:36270ms step_avg:150.50ms
step:252/1480 train_time:36418ms step_avg:150.49ms
step:253/1480 train_time:36566ms step_avg:150.48ms
step:254/1480 train_time:36715ms step_avg:150.47ms
step:255/1480 train_time:36861ms step_avg:150.45ms
step:256/1480 train_time:37010ms step_avg:150.45ms
step:257/1480 train_time:37157ms step_avg:150.43ms
step:258/1480 train_time:37307ms step_avg:150.43ms
step:259/1480 train_time:37456ms step_avg:150.42ms
step:260/1480 train_time:37604ms step_avg:150.42ms
step:261/1480 train_time:37753ms step_avg:150.41ms
step:262/1480 train_time:37900ms step_avg:150.40ms
step:263/1480 train_time:38049ms step_avg:150.39ms
step:264/1480 train_time:38196ms step_avg:150.38ms
step:265/1480 train_time:38344ms step_avg:150.37ms
step:266/1480 train_time:38494ms step_avg:150.37ms
step:267/1480 train_time:38642ms step_avg:150.36ms
step:268/1480 train_time:38791ms step_avg:150.35ms
step:269/1480 train_time:38938ms step_avg:150.34ms
step:270/1480 train_time:39086ms step_avg:150.33ms
step:271/1480 train_time:39236ms step_avg:150.33ms
step:272/1480 train_time:39383ms step_avg:150.32ms
step:273/1480 train_time:39533ms step_avg:150.31ms
step:274/1480 train_time:39681ms step_avg:150.31ms
step:275/1480 train_time:39831ms step_avg:150.30ms
step:276/1480 train_time:39978ms step_avg:150.29ms
step:277/1480 train_time:40127ms step_avg:150.29ms
step:278/1480 train_time:40276ms step_avg:150.28ms
step:279/1480 train_time:40423ms step_avg:150.27ms
step:280/1480 train_time:40572ms step_avg:150.27ms
step:281/1480 train_time:40720ms step_avg:150.26ms
step:282/1480 train_time:40869ms step_avg:150.25ms
step:283/1480 train_time:41017ms step_avg:150.25ms
step:284/1480 train_time:41166ms step_avg:150.24ms
step:285/1480 train_time:41315ms step_avg:150.24ms
step:286/1480 train_time:41462ms step_avg:150.23ms
step:287/1480 train_time:41612ms step_avg:150.22ms
step:288/1480 train_time:41759ms step_avg:150.21ms
step:289/1480 train_time:41906ms step_avg:150.20ms
step:290/1480 train_time:42056ms step_avg:150.20ms
step:291/1480 train_time:42204ms step_avg:150.19ms
step:292/1480 train_time:42353ms step_avg:150.19ms
step:293/1480 train_time:42501ms step_avg:150.18ms
step:294/1480 train_time:42650ms step_avg:150.18ms
step:295/1480 train_time:42798ms step_avg:150.17ms
step:296/1480 train_time:42946ms step_avg:150.16ms
step:297/1480 train_time:43094ms step_avg:150.15ms
step:298/1480 train_time:43241ms step_avg:150.14ms
step:299/1480 train_time:43390ms step_avg:150.14ms
step:300/1480 train_time:43538ms step_avg:150.13ms
step:301/1480 train_time:43686ms step_avg:150.12ms
step:302/1480 train_time:43835ms step_avg:150.12ms
step:303/1480 train_time:43982ms step_avg:150.11ms
step:304/1480 train_time:44132ms step_avg:150.11ms
step:305/1480 train_time:44279ms step_avg:150.10ms
step:306/1480 train_time:44428ms step_avg:150.10ms
step:307/1480 train_time:44576ms step_avg:150.09ms
step:308/1480 train_time:44725ms step_avg:150.08ms
step:309/1480 train_time:44874ms step_avg:150.08ms
step:310/1480 train_time:45023ms step_avg:150.08ms
step:311/1480 train_time:45172ms step_avg:150.07ms
step:312/1480 train_time:45319ms step_avg:150.06ms
step:313/1480 train_time:45468ms step_avg:150.06ms
step:314/1480 train_time:45618ms step_avg:150.06ms
step:315/1480 train_time:45766ms step_avg:150.05ms
step:316/1480 train_time:45914ms step_avg:150.05ms
step:317/1480 train_time:46062ms step_avg:150.04ms
step:318/1480 train_time:46212ms step_avg:150.04ms
step:319/1480 train_time:46359ms step_avg:150.03ms
step:320/1480 train_time:46508ms step_avg:150.03ms
step:321/1480 train_time:46657ms step_avg:150.02ms
step:322/1480 train_time:46806ms step_avg:150.02ms
step:323/1480 train_time:46954ms step_avg:150.01ms
step:324/1480 train_time:47102ms step_avg:150.01ms
step:325/1480 train_time:47252ms step_avg:150.01ms
step:326/1480 train_time:47399ms step_avg:150.00ms
step:327/1480 train_time:47549ms step_avg:150.00ms
step:328/1480 train_time:47696ms step_avg:149.99ms
step:329/1480 train_time:47844ms step_avg:149.98ms
step:330/1480 train_time:47994ms step_avg:149.98ms
step:331/1480 train_time:48144ms step_avg:149.98ms
step:332/1480 train_time:48295ms step_avg:149.99ms
step:333/1480 train_time:48445ms step_avg:149.98ms
step:334/1480 train_time:48596ms step_avg:149.99ms
step:335/1480 train_time:48746ms step_avg:149.99ms
step:336/1480 train_time:48897ms step_avg:149.99ms
step:337/1480 train_time:49047ms step_avg:149.99ms
step:338/1480 train_time:49198ms step_avg:149.99ms
step:339/1480 train_time:49349ms step_avg:150.00ms
step:340/1480 train_time:49499ms step_avg:150.00ms
step:341/1480 train_time:49651ms step_avg:150.00ms
step:342/1480 train_time:49800ms step_avg:150.00ms
step:343/1480 train_time:49952ms step_avg:150.01ms
step:344/1480 train_time:50102ms step_avg:150.01ms
step:345/1480 train_time:50254ms step_avg:150.01ms
step:346/1480 train_time:50404ms step_avg:150.01ms
step:347/1480 train_time:50556ms step_avg:150.02ms
step:348/1480 train_time:50706ms step_avg:150.02ms
step:349/1480 train_time:50857ms step_avg:150.02ms
step:350/1480 train_time:51009ms step_avg:150.03ms
step:351/1480 train_time:51160ms step_avg:150.03ms
step:352/1480 train_time:51311ms step_avg:150.03ms
step:353/1480 train_time:51461ms step_avg:150.03ms
step:354/1480 train_time:51613ms step_avg:150.04ms
step:355/1480 train_time:51764ms step_avg:150.04ms
step:356/1480 train_time:51915ms step_avg:150.04ms
step:357/1480 train_time:52065ms step_avg:150.04ms
step:358/1480 train_time:52216ms step_avg:150.05ms
step:359/1480 train_time:52367ms step_avg:150.05ms
step:360/1480 train_time:52518ms step_avg:150.05ms
step:361/1480 train_time:52669ms step_avg:150.06ms
step:362/1480 train_time:52821ms step_avg:150.06ms
step:363/1480 train_time:52971ms step_avg:150.06ms
step:364/1480 train_time:53122ms step_avg:150.06ms
step:365/1480 train_time:53273ms step_avg:150.06ms
step:366/1480 train_time:53423ms step_avg:150.06ms
step:367/1480 train_time:53574ms step_avg:150.07ms
step:368/1480 train_time:53725ms step_avg:150.07ms
step:369/1480 train_time:53876ms step_avg:150.07ms
step:370/1480 train_time:54027ms step_avg:150.08ms
step:371/1480 train_time:54177ms step_avg:150.08ms
step:372/1480 train_time:54328ms step_avg:150.08ms
step:373/1480 train_time:54479ms step_avg:150.08ms
step:374/1480 train_time:54630ms step_avg:150.08ms
step:375/1480 train_time:54779ms step_avg:150.08ms
step:375/1480 val_loss:3.8112 train_time:54848ms step_avg:150.27ms
step:376/1480 train_time:54939ms step_avg:150.11ms
step:377/1480 train_time:55089ms step_avg:150.11ms
step:378/1480 train_time:55239ms step_avg:150.11ms
step:379/1480 train_time:55405ms step_avg:150.15ms
step:380/1480 train_time:55540ms step_avg:150.11ms
step:381/1480 train_time:55691ms step_avg:150.11ms
step:382/1480 train_time:55841ms step_avg:150.11ms
step:383/1480 train_time:55992ms step_avg:150.11ms
step:384/1480 train_time:56142ms step_avg:150.11ms
step:385/1480 train_time:56294ms step_avg:150.12ms
step:386/1480 train_time:56445ms step_avg:150.12ms
step:387/1480 train_time:56596ms step_avg:150.12ms
step:388/1480 train_time:56747ms step_avg:150.12ms
step:389/1480 train_time:56898ms step_avg:150.13ms
step:390/1480 train_time:57049ms step_avg:150.13ms
step:391/1480 train_time:57200ms step_avg:150.13ms
step:392/1480 train_time:57352ms step_avg:150.14ms
step:393/1480 train_time:57502ms step_avg:150.13ms
step:394/1480 train_time:57653ms step_avg:150.14ms
step:395/1480 train_time:57803ms step_avg:150.14ms
step:396/1480 train_time:57954ms step_avg:150.14ms
step:397/1480 train_time:58105ms step_avg:150.14ms
step:398/1480 train_time:58257ms step_avg:150.15ms
step:399/1480 train_time:58407ms step_avg:150.15ms
step:400/1480 train_time:58558ms step_avg:150.15ms
step:401/1480 train_time:58708ms step_avg:150.15ms
step:402/1480 train_time:58859ms step_avg:150.15ms
step:403/1480 train_time:59011ms step_avg:150.16ms
step:404/1480 train_time:59161ms step_avg:150.16ms
step:405/1480 train_time:59313ms step_avg:150.16ms
step:406/1480 train_time:59462ms step_avg:150.16ms
step:407/1480 train_time:59613ms step_avg:150.16ms
step:408/1480 train_time:59762ms step_avg:150.16ms
step:409/1480 train_time:59914ms step_avg:150.16ms
step:410/1480 train_time:60063ms step_avg:150.16ms
step:411/1480 train_time:60215ms step_avg:150.16ms
step:412/1480 train_time:60366ms step_avg:150.16ms
step:413/1480 train_time:60516ms step_avg:150.16ms
step:414/1480 train_time:60667ms step_avg:150.17ms
step:415/1480 train_time:60818ms step_avg:150.17ms
step:416/1480 train_time:60969ms step_avg:150.17ms
step:417/1480 train_time:61119ms step_avg:150.17ms
step:418/1480 train_time:61272ms step_avg:150.18ms
step:419/1480 train_time:61421ms step_avg:150.17ms
step:420/1480 train_time:61574ms step_avg:150.18ms
step:421/1480 train_time:61724ms step_avg:150.18ms
step:422/1480 train_time:61875ms step_avg:150.18ms
step:423/1480 train_time:62026ms step_avg:150.18ms
step:424/1480 train_time:62177ms step_avg:150.19ms
step:425/1480 train_time:62329ms step_avg:150.19ms
step:426/1480 train_time:62480ms step_avg:150.19ms
step:427/1480 train_time:62631ms step_avg:150.19ms
step:428/1480 train_time:62780ms step_avg:150.19ms
step:429/1480 train_time:62932ms step_avg:150.20ms
step:430/1480 train_time:63082ms step_avg:150.20ms
step:431/1480 train_time:63234ms step_avg:150.20ms
step:432/1480 train_time:63383ms step_avg:150.20ms
step:433/1480 train_time:63535ms step_avg:150.20ms
step:434/1480 train_time:63685ms step_avg:150.20ms
step:435/1480 train_time:63836ms step_avg:150.20ms
step:436/1480 train_time:63986ms step_avg:150.20ms
step:437/1480 train_time:64137ms step_avg:150.20ms
step:438/1480 train_time:64288ms step_avg:150.21ms
step:439/1480 train_time:64441ms step_avg:150.21ms
step:440/1480 train_time:64592ms step_avg:150.21ms
step:441/1480 train_time:64744ms step_avg:150.22ms
step:442/1480 train_time:64898ms step_avg:150.23ms
step:443/1480 train_time:65051ms step_avg:150.23ms
step:444/1480 train_time:65203ms step_avg:150.24ms
step:445/1480 train_time:65356ms step_avg:150.24ms
step:446/1480 train_time:65508ms step_avg:150.25ms
step:447/1480 train_time:65660ms step_avg:150.25ms
step:448/1480 train_time:65813ms step_avg:150.26ms
step:449/1480 train_time:65965ms step_avg:150.26ms
step:450/1480 train_time:66118ms step_avg:150.27ms
step:451/1480 train_time:66272ms step_avg:150.28ms
step:452/1480 train_time:66424ms step_avg:150.28ms
step:453/1480 train_time:66577ms step_avg:150.29ms
step:454/1480 train_time:66730ms step_avg:150.29ms
step:455/1480 train_time:66882ms step_avg:150.30ms
step:456/1480 train_time:67035ms step_avg:150.30ms
step:457/1480 train_time:67187ms step_avg:150.31ms
step:458/1480 train_time:67340ms step_avg:150.31ms
step:459/1480 train_time:67492ms step_avg:150.32ms
step:460/1480 train_time:67645ms step_avg:150.32ms
step:461/1480 train_time:67797ms step_avg:150.33ms
step:462/1480 train_time:67951ms step_avg:150.33ms
step:463/1480 train_time:68104ms step_avg:150.34ms
step:464/1480 train_time:68256ms step_avg:150.34ms
step:465/1480 train_time:68409ms step_avg:150.35ms
step:466/1480 train_time:68560ms step_avg:150.35ms
step:467/1480 train_time:68713ms step_avg:150.36ms
step:468/1480 train_time:68865ms step_avg:150.36ms
step:469/1480 train_time:69017ms step_avg:150.36ms
step:470/1480 train_time:69170ms step_avg:150.37ms
step:471/1480 train_time:69323ms step_avg:150.38ms
step:472/1480 train_time:69476ms step_avg:150.38ms
step:473/1480 train_time:69628ms step_avg:150.39ms
step:474/1480 train_time:69781ms step_avg:150.39ms
step:475/1480 train_time:69933ms step_avg:150.39ms
step:476/1480 train_time:70086ms step_avg:150.40ms
step:477/1480 train_time:70239ms step_avg:150.41ms
step:478/1480 train_time:70394ms step_avg:150.41ms
step:479/1480 train_time:70548ms step_avg:150.42ms
step:480/1480 train_time:70700ms step_avg:150.43ms
step:481/1480 train_time:70854ms step_avg:150.43ms
step:482/1480 train_time:71006ms step_avg:150.44ms
step:483/1480 train_time:71158ms step_avg:150.44ms
step:484/1480 train_time:71312ms step_avg:150.45ms
step:485/1480 train_time:71463ms step_avg:150.45ms
step:486/1480 train_time:71617ms step_avg:150.46ms
step:487/1480 train_time:71769ms step_avg:150.46ms
step:488/1480 train_time:71921ms step_avg:150.46ms
step:489/1480 train_time:72074ms step_avg:150.47ms
step:490/1480 train_time:72227ms step_avg:150.47ms
step:491/1480 train_time:72380ms step_avg:150.48ms
step:492/1480 train_time:72533ms step_avg:150.48ms
step:493/1480 train_time:72686ms step_avg:150.49ms
step:494/1480 train_time:72839ms step_avg:150.49ms
step:495/1480 train_time:72992ms step_avg:150.50ms
step:496/1480 train_time:73147ms step_avg:150.51ms
step:497/1480 train_time:73300ms step_avg:150.51ms
step:498/1480 train_time:73454ms step_avg:150.52ms
step:499/1480 train_time:73607ms step_avg:150.52ms
step:500/1480 train_time:73760ms step_avg:150.53ms
step:500/1480 val_loss:3.6842 train_time:73830ms step_avg:150.67ms
step:501/1480 train_time:73925ms step_avg:150.56ms
step:502/1480 train_time:74071ms step_avg:150.55ms
step:503/1480 train_time:74224ms step_avg:150.56ms
step:504/1480 train_time:74376ms step_avg:150.56ms
step:505/1480 train_time:74527ms step_avg:150.56ms
step:506/1480 train_time:74680ms step_avg:150.56ms
step:507/1480 train_time:74832ms step_avg:150.57ms
step:508/1480 train_time:74987ms step_avg:150.58ms
step:509/1480 train_time:75141ms step_avg:150.58ms
step:510/1480 train_time:75293ms step_avg:150.59ms
step:511/1480 train_time:75446ms step_avg:150.59ms
step:512/1480 train_time:75599ms step_avg:150.60ms
step:513/1480 train_time:75751ms step_avg:150.60ms
step:514/1480 train_time:75905ms step_avg:150.60ms
step:515/1480 train_time:76060ms step_avg:150.61ms
step:516/1480 train_time:76213ms step_avg:150.62ms
step:517/1480 train_time:76366ms step_avg:150.62ms
step:518/1480 train_time:76519ms step_avg:150.63ms
step:519/1480 train_time:76671ms step_avg:150.63ms
step:520/1480 train_time:76824ms step_avg:150.64ms
step:521/1480 train_time:76976ms step_avg:150.64ms
step:522/1480 train_time:77130ms step_avg:150.64ms
step:523/1480 train_time:77283ms step_avg:150.65ms
step:524/1480 train_time:77435ms step_avg:150.65ms
step:525/1480 train_time:77587ms step_avg:150.65ms
step:526/1480 train_time:77741ms step_avg:150.66ms
step:527/1480 train_time:77892ms step_avg:150.66ms
step:528/1480 train_time:78044ms step_avg:150.66ms
step:529/1480 train_time:78197ms step_avg:150.67ms
step:530/1480 train_time:78351ms step_avg:150.67ms
step:531/1480 train_time:78504ms step_avg:150.68ms
step:532/1480 train_time:78658ms step_avg:150.69ms
step:533/1480 train_time:78810ms step_avg:150.69ms
step:534/1480 train_time:78963ms step_avg:150.69ms
step:535/1480 train_time:79115ms step_avg:150.70ms
step:536/1480 train_time:79267ms step_avg:150.70ms
step:537/1480 train_time:79421ms step_avg:150.70ms
step:538/1480 train_time:79575ms step_avg:150.71ms
step:539/1480 train_time:79727ms step_avg:150.71ms
step:540/1480 train_time:79880ms step_avg:150.72ms
step:541/1480 train_time:80033ms step_avg:150.72ms
step:542/1480 train_time:80185ms step_avg:150.72ms
step:543/1480 train_time:80339ms step_avg:150.73ms
step:544/1480 train_time:80491ms step_avg:150.73ms
step:545/1480 train_time:80644ms step_avg:150.74ms
step:546/1480 train_time:80796ms step_avg:150.74ms
step:547/1480 train_time:80950ms step_avg:150.74ms
step:548/1480 train_time:81103ms step_avg:150.75ms
step:549/1480 train_time:81256ms step_avg:150.75ms
step:550/1480 train_time:81409ms step_avg:150.76ms
step:551/1480 train_time:81563ms step_avg:150.76ms
step:552/1480 train_time:81717ms step_avg:150.77ms
step:553/1480 train_time:81872ms step_avg:150.78ms
step:554/1480 train_time:82026ms step_avg:150.78ms
step:555/1480 train_time:82181ms step_avg:150.79ms
step:556/1480 train_time:82336ms step_avg:150.80ms
step:557/1480 train_time:82491ms step_avg:150.81ms
step:558/1480 train_time:82646ms step_avg:150.81ms
step:559/1480 train_time:82800ms step_avg:150.82ms
step:560/1480 train_time:82956ms step_avg:150.83ms
step:561/1480 train_time:83112ms step_avg:150.84ms
step:562/1480 train_time:83266ms step_avg:150.84ms
step:563/1480 train_time:83421ms step_avg:150.85ms
step:564/1480 train_time:83576ms step_avg:150.86ms
step:565/1480 train_time:83731ms step_avg:150.87ms
step:566/1480 train_time:83886ms step_avg:150.87ms
step:567/1480 train_time:84040ms step_avg:150.88ms
step:568/1480 train_time:84194ms step_avg:150.89ms
step:569/1480 train_time:84361ms step_avg:150.91ms
step:570/1480 train_time:84503ms step_avg:150.90ms
step:571/1480 train_time:84658ms step_avg:150.91ms
step:572/1480 train_time:84812ms step_avg:150.91ms
step:573/1480 train_time:84966ms step_avg:150.92ms
step:574/1480 train_time:85124ms step_avg:150.93ms
step:575/1480 train_time:85280ms step_avg:150.94ms
step:576/1480 train_time:85435ms step_avg:150.94ms
step:577/1480 train_time:85590ms step_avg:150.95ms
step:578/1480 train_time:85744ms step_avg:150.96ms
step:579/1480 train_time:85898ms step_avg:150.96ms
step:580/1480 train_time:86054ms step_avg:150.97ms
step:581/1480 train_time:86209ms step_avg:150.98ms
step:582/1480 train_time:86363ms step_avg:150.98ms
step:583/1480 train_time:86518ms step_avg:150.99ms
step:584/1480 train_time:86672ms step_avg:151.00ms
step:585/1480 train_time:86827ms step_avg:151.00ms
step:586/1480 train_time:86983ms step_avg:151.01ms
step:587/1480 train_time:87138ms step_avg:151.02ms
step:588/1480 train_time:87292ms step_avg:151.02ms
step:589/1480 train_time:87447ms step_avg:151.03ms
step:590/1480 train_time:87601ms step_avg:151.04ms
step:591/1480 train_time:87756ms step_avg:151.04ms
step:592/1480 train_time:87912ms step_avg:151.05ms
step:593/1480 train_time:88067ms step_avg:151.06ms
step:594/1480 train_time:88222ms step_avg:151.07ms
step:595/1480 train_time:88378ms step_avg:151.07ms
step:596/1480 train_time:88533ms step_avg:151.08ms
step:597/1480 train_time:88689ms step_avg:151.09ms
step:598/1480 train_time:88843ms step_avg:151.09ms
step:599/1480 train_time:88996ms step_avg:151.10ms
step:600/1480 train_time:89150ms step_avg:151.10ms
step:601/1480 train_time:89305ms step_avg:151.11ms
step:602/1480 train_time:89460ms step_avg:151.11ms
step:603/1480 train_time:89615ms step_avg:151.12ms
step:604/1480 train_time:89769ms step_avg:151.13ms
step:605/1480 train_time:89925ms step_avg:151.13ms
step:606/1480 train_time:90080ms step_avg:151.14ms
step:607/1480 train_time:90235ms step_avg:151.15ms
step:608/1480 train_time:90391ms step_avg:151.16ms
step:609/1480 train_time:90545ms step_avg:151.16ms
step:610/1480 train_time:90699ms step_avg:151.16ms
step:611/1480 train_time:90854ms step_avg:151.17ms
step:612/1480 train_time:91010ms step_avg:151.18ms
step:613/1480 train_time:91165ms step_avg:151.19ms
step:614/1480 train_time:91320ms step_avg:151.19ms
step:615/1480 train_time:91474ms step_avg:151.20ms
step:616/1480 train_time:91627ms step_avg:151.20ms
step:617/1480 train_time:91782ms step_avg:151.21ms
step:618/1480 train_time:91937ms step_avg:151.21ms
step:619/1480 train_time:92092ms step_avg:151.22ms
step:620/1480 train_time:92247ms step_avg:151.22ms
step:621/1480 train_time:92401ms step_avg:151.23ms
step:622/1480 train_time:92556ms step_avg:151.24ms
step:623/1480 train_time:92711ms step_avg:151.24ms
step:624/1480 train_time:92866ms step_avg:151.25ms
step:625/1480 train_time:93021ms step_avg:151.25ms
step:625/1480 val_loss:3.6035 train_time:93090ms step_avg:151.37ms
step:626/1480 train_time:93182ms step_avg:151.27ms
step:627/1480 train_time:93336ms step_avg:151.27ms
step:628/1480 train_time:93490ms step_avg:151.28ms
step:629/1480 train_time:93644ms step_avg:151.28ms
step:630/1480 train_time:93797ms step_avg:151.29ms
step:631/1480 train_time:93952ms step_avg:151.29ms
step:632/1480 train_time:94107ms step_avg:151.30ms
step:633/1480 train_time:94262ms step_avg:151.30ms
step:634/1480 train_time:94418ms step_avg:151.31ms
step:635/1480 train_time:94571ms step_avg:151.31ms
step:636/1480 train_time:94726ms step_avg:151.32ms
step:637/1480 train_time:94880ms step_avg:151.32ms
step:638/1480 train_time:95035ms step_avg:151.33ms
step:639/1480 train_time:95189ms step_avg:151.33ms
step:640/1480 train_time:95343ms step_avg:151.34ms
step:641/1480 train_time:95497ms step_avg:151.34ms
step:642/1480 train_time:95652ms step_avg:151.35ms
step:643/1480 train_time:95806ms step_avg:151.35ms
step:644/1480 train_time:95961ms step_avg:151.36ms
step:645/1480 train_time:96116ms step_avg:151.36ms
step:646/1480 train_time:96270ms step_avg:151.37ms
step:647/1480 train_time:96425ms step_avg:151.37ms
step:648/1480 train_time:96582ms step_avg:151.38ms
step:649/1480 train_time:96737ms step_avg:151.39ms
step:650/1480 train_time:96891ms step_avg:151.39ms
step:651/1480 train_time:97046ms step_avg:151.40ms
step:652/1480 train_time:97202ms step_avg:151.40ms
step:653/1480 train_time:97356ms step_avg:151.41ms
step:654/1480 train_time:97511ms step_avg:151.41ms
step:655/1480 train_time:97665ms step_avg:151.42ms
step:656/1480 train_time:97820ms step_avg:151.42ms
step:657/1480 train_time:97975ms step_avg:151.43ms
step:658/1480 train_time:98130ms step_avg:151.44ms
step:659/1480 train_time:98285ms step_avg:151.44ms
step:660/1480 train_time:98442ms step_avg:151.45ms
step:661/1480 train_time:98598ms step_avg:151.46ms
step:662/1480 train_time:98754ms step_avg:151.46ms
step:663/1480 train_time:98909ms step_avg:151.47ms
step:664/1480 train_time:99065ms step_avg:151.48ms
step:665/1480 train_time:99223ms step_avg:151.48ms
step:666/1480 train_time:99378ms step_avg:151.49ms
step:667/1480 train_time:99535ms step_avg:151.50ms
step:668/1480 train_time:99691ms step_avg:151.51ms
step:669/1480 train_time:99850ms step_avg:151.52ms
step:670/1480 train_time:100006ms step_avg:151.52ms
step:671/1480 train_time:100161ms step_avg:151.53ms
step:672/1480 train_time:100317ms step_avg:151.54ms
step:673/1480 train_time:100472ms step_avg:151.54ms
step:674/1480 train_time:100629ms step_avg:151.55ms
step:675/1480 train_time:100787ms step_avg:151.56ms
step:676/1480 train_time:100945ms step_avg:151.57ms
step:677/1480 train_time:101101ms step_avg:151.58ms
step:678/1480 train_time:101257ms step_avg:151.58ms
step:679/1480 train_time:101413ms step_avg:151.59ms
step:680/1480 train_time:101569ms step_avg:151.60ms
step:681/1480 train_time:101725ms step_avg:151.60ms
step:682/1480 train_time:101884ms step_avg:151.61ms
step:683/1480 train_time:102041ms step_avg:151.62ms
step:684/1480 train_time:102196ms step_avg:151.63ms
step:685/1480 train_time:102354ms step_avg:151.64ms
step:686/1480 train_time:102509ms step_avg:151.64ms
step:687/1480 train_time:102665ms step_avg:151.65ms
step:688/1480 train_time:102823ms step_avg:151.66ms
step:689/1480 train_time:102980ms step_avg:151.66ms
step:690/1480 train_time:103138ms step_avg:151.67ms
step:691/1480 train_time:103294ms step_avg:151.68ms
step:692/1480 train_time:103450ms step_avg:151.69ms
step:693/1480 train_time:103606ms step_avg:151.69ms
step:694/1480 train_time:103762ms step_avg:151.70ms
step:695/1480 train_time:103918ms step_avg:151.71ms
step:696/1480 train_time:104074ms step_avg:151.71ms
step:697/1480 train_time:104231ms step_avg:151.72ms
step:698/1480 train_time:104387ms step_avg:151.73ms
step:699/1480 train_time:104544ms step_avg:151.73ms
step:700/1480 train_time:104699ms step_avg:151.74ms
step:701/1480 train_time:104856ms step_avg:151.75ms
step:702/1480 train_time:105013ms step_avg:151.75ms
step:703/1480 train_time:105168ms step_avg:151.76ms
step:704/1480 train_time:105324ms step_avg:151.76ms
step:705/1480 train_time:105481ms step_avg:151.77ms
step:706/1480 train_time:105639ms step_avg:151.78ms
step:707/1480 train_time:105796ms step_avg:151.79ms
step:708/1480 train_time:105952ms step_avg:151.79ms
step:709/1480 train_time:106108ms step_avg:151.80ms
step:710/1480 train_time:106263ms step_avg:151.80ms
step:711/1480 train_time:106419ms step_avg:151.81ms
step:712/1480 train_time:106576ms step_avg:151.82ms
step:713/1480 train_time:106733ms step_avg:151.82ms
step:714/1480 train_time:106890ms step_avg:151.83ms
step:715/1480 train_time:107046ms step_avg:151.84ms
step:716/1480 train_time:107200ms step_avg:151.84ms
step:717/1480 train_time:107357ms step_avg:151.85ms
step:718/1480 train_time:107513ms step_avg:151.85ms
step:719/1480 train_time:107668ms step_avg:151.86ms
step:720/1480 train_time:107827ms step_avg:151.87ms
step:721/1480 train_time:107985ms step_avg:151.88ms
step:722/1480 train_time:108141ms step_avg:151.88ms
step:723/1480 train_time:108297ms step_avg:151.89ms
step:724/1480 train_time:108453ms step_avg:151.90ms
step:725/1480 train_time:108610ms step_avg:151.90ms
step:726/1480 train_time:108765ms step_avg:151.91ms
step:727/1480 train_time:108923ms step_avg:151.92ms
step:728/1480 train_time:109078ms step_avg:151.92ms
step:729/1480 train_time:109236ms step_avg:151.93ms
step:730/1480 train_time:109392ms step_avg:151.93ms
step:731/1480 train_time:109550ms step_avg:151.94ms
step:732/1480 train_time:109706ms step_avg:151.95ms
step:733/1480 train_time:109862ms step_avg:151.95ms
step:734/1480 train_time:110018ms step_avg:151.96ms
step:735/1480 train_time:110175ms step_avg:151.97ms
step:736/1480 train_time:110333ms step_avg:151.97ms
step:737/1480 train_time:110489ms step_avg:151.98ms
step:738/1480 train_time:110644ms step_avg:151.98ms
step:739/1480 train_time:110800ms step_avg:151.99ms
step:740/1480 train_time:110958ms step_avg:152.00ms
step:741/1480 train_time:111115ms step_avg:152.00ms
step:742/1480 train_time:111272ms step_avg:152.01ms
step:743/1480 train_time:111427ms step_avg:152.02ms
step:744/1480 train_time:111583ms step_avg:152.02ms
step:745/1480 train_time:111740ms step_avg:152.03ms
step:746/1480 train_time:111896ms step_avg:152.03ms
step:747/1480 train_time:112052ms step_avg:152.04ms
step:748/1480 train_time:112212ms step_avg:152.05ms
step:749/1480 train_time:112369ms step_avg:152.06ms
step:750/1480 train_time:112524ms step_avg:152.06ms
step:750/1480 val_loss:3.5464 train_time:112598ms step_avg:152.16ms
step:751/1480 train_time:112688ms step_avg:152.08ms
step:752/1480 train_time:112844ms step_avg:152.08ms
step:753/1480 train_time:112999ms step_avg:152.08ms
step:754/1480 train_time:113154ms step_avg:152.09ms
step:755/1480 train_time:113310ms step_avg:152.09ms
step:756/1480 train_time:113466ms step_avg:152.10ms
step:757/1480 train_time:113624ms step_avg:152.11ms
step:758/1480 train_time:113780ms step_avg:152.11ms
step:759/1480 train_time:113948ms step_avg:152.13ms
step:760/1480 train_time:114096ms step_avg:152.13ms
step:761/1480 train_time:114252ms step_avg:152.13ms
step:762/1480 train_time:114408ms step_avg:152.14ms
step:763/1480 train_time:114565ms step_avg:152.14ms
step:764/1480 train_time:114721ms step_avg:152.15ms
step:765/1480 train_time:114878ms step_avg:152.16ms
step:766/1480 train_time:115036ms step_avg:152.16ms
step:767/1480 train_time:115194ms step_avg:152.17ms
step:768/1480 train_time:115349ms step_avg:152.18ms
step:769/1480 train_time:115507ms step_avg:152.18ms
step:770/1480 train_time:115664ms step_avg:152.19ms
step:771/1480 train_time:115821ms step_avg:152.20ms
step:772/1480 train_time:115977ms step_avg:152.20ms
step:773/1480 train_time:116136ms step_avg:152.21ms
step:774/1480 train_time:116295ms step_avg:152.22ms
step:775/1480 train_time:116452ms step_avg:152.23ms
step:776/1480 train_time:116612ms step_avg:152.24ms
step:777/1480 train_time:116772ms step_avg:152.25ms
step:778/1480 train_time:116931ms step_avg:152.25ms
step:779/1480 train_time:117088ms step_avg:152.26ms
step:780/1480 train_time:117246ms step_avg:152.27ms
step:781/1480 train_time:117403ms step_avg:152.27ms
step:782/1480 train_time:117560ms step_avg:152.28ms
step:783/1480 train_time:117717ms step_avg:152.29ms
step:784/1480 train_time:117877ms step_avg:152.30ms
step:785/1480 train_time:118038ms step_avg:152.31ms
step:786/1480 train_time:118196ms step_avg:152.31ms
step:787/1480 train_time:118354ms step_avg:152.32ms
step:788/1480 train_time:118512ms step_avg:152.33ms
step:789/1480 train_time:118669ms step_avg:152.34ms
step:790/1480 train_time:118827ms step_avg:152.34ms
step:791/1480 train_time:118985ms step_avg:152.35ms
step:792/1480 train_time:119144ms step_avg:152.36ms
step:793/1480 train_time:119300ms step_avg:152.36ms
step:794/1480 train_time:119458ms step_avg:152.37ms
step:795/1480 train_time:119617ms step_avg:152.38ms
step:796/1480 train_time:119779ms step_avg:152.39ms
step:797/1480 train_time:119940ms step_avg:152.40ms
step:798/1480 train_time:120100ms step_avg:152.41ms
step:799/1480 train_time:120262ms step_avg:152.42ms
step:800/1480 train_time:120420ms step_avg:152.43ms
step:801/1480 train_time:120577ms step_avg:152.44ms
step:802/1480 train_time:120738ms step_avg:152.45ms
step:803/1480 train_time:120898ms step_avg:152.46ms
step:804/1480 train_time:121055ms step_avg:152.46ms
step:805/1480 train_time:121216ms step_avg:152.47ms
step:806/1480 train_time:121374ms step_avg:152.48ms
step:807/1480 train_time:121531ms step_avg:152.49ms
step:808/1480 train_time:121688ms step_avg:152.49ms
step:809/1480 train_time:121845ms step_avg:152.50ms
step:810/1480 train_time:122002ms step_avg:152.50ms
step:811/1480 train_time:122159ms step_avg:152.51ms
step:812/1480 train_time:122317ms step_avg:152.51ms
step:813/1480 train_time:122474ms step_avg:152.52ms
step:814/1480 train_time:122631ms step_avg:152.53ms
step:815/1480 train_time:122788ms step_avg:152.53ms
step:816/1480 train_time:122947ms step_avg:152.54ms
step:817/1480 train_time:123103ms step_avg:152.54ms
step:818/1480 train_time:123260ms step_avg:152.55ms
step:819/1480 train_time:123420ms step_avg:152.56ms
step:820/1480 train_time:123578ms step_avg:152.57ms
step:821/1480 train_time:123736ms step_avg:152.57ms
step:822/1480 train_time:123894ms step_avg:152.58ms
step:823/1480 train_time:124052ms step_avg:152.59ms
step:824/1480 train_time:124209ms step_avg:152.59ms
step:825/1480 train_time:124367ms step_avg:152.60ms
step:826/1480 train_time:124527ms step_avg:152.61ms
step:827/1480 train_time:124684ms step_avg:152.61ms
step:828/1480 train_time:124844ms step_avg:152.62ms
step:829/1480 train_time:125002ms step_avg:152.63ms
step:830/1480 train_time:125161ms step_avg:152.63ms
step:831/1480 train_time:125319ms step_avg:152.64ms
step:832/1480 train_time:125477ms step_avg:152.65ms
step:833/1480 train_time:125635ms step_avg:152.66ms
step:834/1480 train_time:125795ms step_avg:152.66ms
step:835/1480 train_time:125954ms step_avg:152.67ms
step:836/1480 train_time:126113ms step_avg:152.68ms
step:837/1480 train_time:126269ms step_avg:152.68ms
step:838/1480 train_time:126427ms step_avg:152.69ms
step:839/1480 train_time:126584ms step_avg:152.69ms
step:840/1480 train_time:126743ms step_avg:152.70ms
step:841/1480 train_time:126900ms step_avg:152.71ms
step:842/1480 train_time:127058ms step_avg:152.71ms
step:843/1480 train_time:127215ms step_avg:152.72ms
step:844/1480 train_time:127371ms step_avg:152.72ms
step:845/1480 train_time:127528ms step_avg:152.73ms
step:846/1480 train_time:127688ms step_avg:152.74ms
step:847/1480 train_time:127846ms step_avg:152.74ms
step:848/1480 train_time:128003ms step_avg:152.75ms
step:849/1480 train_time:128161ms step_avg:152.75ms
step:850/1480 train_time:128318ms step_avg:152.76ms
step:851/1480 train_time:128477ms step_avg:152.77ms
step:852/1480 train_time:128635ms step_avg:152.77ms
step:853/1480 train_time:128793ms step_avg:152.78ms
step:854/1480 train_time:128950ms step_avg:152.78ms
step:855/1480 train_time:129108ms step_avg:152.79ms
step:856/1480 train_time:129265ms step_avg:152.80ms
step:857/1480 train_time:129423ms step_avg:152.80ms
step:858/1480 train_time:129581ms step_avg:152.81ms
step:859/1480 train_time:129741ms step_avg:152.82ms
step:860/1480 train_time:129899ms step_avg:152.82ms
step:861/1480 train_time:130058ms step_avg:152.83ms
step:862/1480 train_time:130222ms step_avg:152.84ms
step:863/1480 train_time:130382ms step_avg:152.85ms
step:864/1480 train_time:130541ms step_avg:152.86ms
step:865/1480 train_time:130698ms step_avg:152.86ms
step:866/1480 train_time:130857ms step_avg:152.87ms
step:867/1480 train_time:131016ms step_avg:152.88ms
step:868/1480 train_time:131173ms step_avg:152.88ms
step:869/1480 train_time:131330ms step_avg:152.89ms
step:870/1480 train_time:131489ms step_avg:152.89ms
step:871/1480 train_time:131645ms step_avg:152.90ms
step:872/1480 train_time:131803ms step_avg:152.90ms
step:873/1480 train_time:131960ms step_avg:152.91ms
step:874/1480 train_time:132120ms step_avg:152.92ms
step:875/1480 train_time:132280ms step_avg:152.92ms
step:875/1480 val_loss:3.5010 train_time:132353ms step_avg:153.01ms
step:876/1480 train_time:132447ms step_avg:152.94ms
step:877/1480 train_time:132601ms step_avg:152.94ms
step:878/1480 train_time:132758ms step_avg:152.95ms
step:879/1480 train_time:132918ms step_avg:152.96ms
step:880/1480 train_time:133077ms step_avg:152.96ms
step:881/1480 train_time:133234ms step_avg:152.97ms
step:882/1480 train_time:133395ms step_avg:152.98ms
step:883/1480 train_time:133556ms step_avg:152.99ms
step:884/1480 train_time:133717ms step_avg:152.99ms
step:885/1480 train_time:133877ms step_avg:153.00ms
step:886/1480 train_time:134038ms step_avg:153.01ms
step:887/1480 train_time:134198ms step_avg:153.02ms
step:888/1480 train_time:134362ms step_avg:153.03ms
step:889/1480 train_time:134523ms step_avg:153.04ms
step:890/1480 train_time:134681ms step_avg:153.05ms
step:891/1480 train_time:134840ms step_avg:153.05ms
step:892/1480 train_time:135000ms step_avg:153.06ms
step:893/1480 train_time:135159ms step_avg:153.07ms
step:894/1480 train_time:135319ms step_avg:153.08ms
step:895/1480 train_time:135479ms step_avg:153.08ms
step:896/1480 train_time:135637ms step_avg:153.09ms
step:897/1480 train_time:135798ms step_avg:153.10ms
step:898/1480 train_time:135959ms step_avg:153.11ms
step:899/1480 train_time:136119ms step_avg:153.11ms
step:900/1480 train_time:136278ms step_avg:153.12ms
step:901/1480 train_time:136437ms step_avg:153.13ms
step:902/1480 train_time:136596ms step_avg:153.13ms
step:903/1480 train_time:136757ms step_avg:153.14ms
step:904/1480 train_time:136918ms step_avg:153.15ms
step:905/1480 train_time:137077ms step_avg:153.16ms
step:906/1480 train_time:137236ms step_avg:153.17ms
step:907/1480 train_time:137398ms step_avg:153.17ms
step:908/1480 train_time:137556ms step_avg:153.18ms
step:909/1480 train_time:137715ms step_avg:153.19ms
step:910/1480 train_time:137880ms step_avg:153.20ms
step:911/1480 train_time:138039ms step_avg:153.21ms
step:912/1480 train_time:138200ms step_avg:153.21ms
step:913/1480 train_time:138361ms step_avg:153.22ms
step:914/1480 train_time:138521ms step_avg:153.23ms
step:915/1480 train_time:138684ms step_avg:153.24ms
step:916/1480 train_time:138842ms step_avg:153.25ms
step:917/1480 train_time:139001ms step_avg:153.25ms
step:918/1480 train_time:139162ms step_avg:153.26ms
step:919/1480 train_time:139324ms step_avg:153.27ms
step:920/1480 train_time:139483ms step_avg:153.28ms
step:921/1480 train_time:139641ms step_avg:153.28ms
step:922/1480 train_time:139803ms step_avg:153.29ms
step:923/1480 train_time:139960ms step_avg:153.30ms
step:924/1480 train_time:140120ms step_avg:153.30ms
step:925/1480 train_time:140279ms step_avg:153.31ms
step:926/1480 train_time:140437ms step_avg:153.32ms
step:927/1480 train_time:140595ms step_avg:153.32ms
step:928/1480 train_time:140754ms step_avg:153.33ms
step:929/1480 train_time:140916ms step_avg:153.34ms
step:930/1480 train_time:141075ms step_avg:153.34ms
step:931/1480 train_time:141233ms step_avg:153.35ms
step:932/1480 train_time:141393ms step_avg:153.35ms
step:933/1480 train_time:141551ms step_avg:153.36ms
step:934/1480 train_time:141711ms step_avg:153.37ms
step:935/1480 train_time:141870ms step_avg:153.37ms
step:936/1480 train_time:142027ms step_avg:153.38ms
step:937/1480 train_time:142187ms step_avg:153.38ms
step:938/1480 train_time:142345ms step_avg:153.39ms
step:939/1480 train_time:142507ms step_avg:153.40ms
step:940/1480 train_time:142667ms step_avg:153.40ms
step:941/1480 train_time:142824ms step_avg:153.41ms
step:942/1480 train_time:142983ms step_avg:153.41ms
step:943/1480 train_time:143143ms step_avg:153.42ms
step:944/1480 train_time:143305ms step_avg:153.43ms
step:945/1480 train_time:143463ms step_avg:153.44ms
step:946/1480 train_time:143625ms step_avg:153.45ms
step:947/1480 train_time:143785ms step_avg:153.45ms
step:948/1480 train_time:143944ms step_avg:153.46ms
step:949/1480 train_time:144112ms step_avg:153.47ms
step:950/1480 train_time:144262ms step_avg:153.47ms
step:951/1480 train_time:144422ms step_avg:153.48ms
step:952/1480 train_time:144581ms step_avg:153.48ms
step:953/1480 train_time:144741ms step_avg:153.49ms
step:954/1480 train_time:144904ms step_avg:153.50ms
step:955/1480 train_time:145061ms step_avg:153.50ms
step:956/1480 train_time:145221ms step_avg:153.51ms
step:957/1480 train_time:145381ms step_avg:153.52ms
step:958/1480 train_time:145544ms step_avg:153.53ms
step:959/1480 train_time:145702ms step_avg:153.53ms
step:960/1480 train_time:145862ms step_avg:153.54ms
step:961/1480 train_time:146021ms step_avg:153.54ms
step:962/1480 train_time:146183ms step_avg:153.55ms
step:963/1480 train_time:146343ms step_avg:153.56ms
step:964/1480 train_time:146504ms step_avg:153.57ms
step:965/1480 train_time:146662ms step_avg:153.57ms
step:966/1480 train_time:146821ms step_avg:153.58ms
step:967/1480 train_time:146979ms step_avg:153.58ms
step:968/1480 train_time:147138ms step_avg:153.59ms
step:969/1480 train_time:147300ms step_avg:153.60ms
step:970/1480 train_time:147459ms step_avg:153.60ms
step:971/1480 train_time:147619ms step_avg:153.61ms
step:972/1480 train_time:147777ms step_avg:153.61ms
step:973/1480 train_time:147934ms step_avg:153.62ms
step:974/1480 train_time:148096ms step_avg:153.63ms
step:975/1480 train_time:148258ms step_avg:153.63ms
step:976/1480 train_time:148418ms step_avg:153.64ms
step:977/1480 train_time:148578ms step_avg:153.65ms
step:978/1480 train_time:148738ms step_avg:153.65ms
step:979/1480 train_time:148899ms step_avg:153.66ms
step:980/1480 train_time:149059ms step_avg:153.67ms
step:981/1480 train_time:149220ms step_avg:153.68ms
step:982/1480 train_time:149378ms step_avg:153.68ms
step:983/1480 train_time:149538ms step_avg:153.69ms
step:984/1480 train_time:149697ms step_avg:153.69ms
step:985/1480 train_time:149860ms step_avg:153.70ms
step:986/1480 train_time:150020ms step_avg:153.71ms
step:987/1480 train_time:150179ms step_avg:153.71ms
step:988/1480 train_time:150339ms step_avg:153.72ms
step:989/1480 train_time:150498ms step_avg:153.73ms
step:990/1480 train_time:150661ms step_avg:153.74ms
step:991/1480 train_time:150822ms step_avg:153.74ms
step:992/1480 train_time:150986ms step_avg:153.75ms
step:993/1480 train_time:151155ms step_avg:153.77ms
step:994/1480 train_time:151315ms step_avg:153.78ms
step:995/1480 train_time:151475ms step_avg:153.78ms
step:996/1480 train_time:151632ms step_avg:153.78ms
step:997/1480 train_time:151792ms step_avg:153.79ms
step:998/1480 train_time:151951ms step_avg:153.80ms
step:999/1480 train_time:152113ms step_avg:153.80ms
step:1000/1480 train_time:152276ms step_avg:153.81ms
step:1000/1480 val_loss:3.4375 train_time:152348ms step_avg:153.89ms
step:1001/1480 train_time:152439ms step_avg:153.82ms
step:1002/1480 train_time:152599ms step_avg:153.83ms
step:1003/1480 train_time:152763ms step_avg:153.84ms
step:1004/1480 train_time:152923ms step_avg:153.85ms
step:1005/1480 train_time:153085ms step_avg:153.85ms
step:1006/1480 train_time:153246ms step_avg:153.86ms
step:1007/1480 train_time:153405ms step_avg:153.87ms
step:1008/1480 train_time:153565ms step_avg:153.87ms
step:1009/1480 train_time:153729ms step_avg:153.88ms
step:1010/1480 train_time:153888ms step_avg:153.89ms
step:1011/1480 train_time:154047ms step_avg:153.89ms
step:1012/1480 train_time:154205ms step_avg:153.90ms
step:1013/1480 train_time:154367ms step_avg:153.91ms
step:1014/1480 train_time:154526ms step_avg:153.91ms
step:1015/1480 train_time:154690ms step_avg:153.92ms
step:1016/1480 train_time:154848ms step_avg:153.92ms
step:1017/1480 train_time:155010ms step_avg:153.93ms
step:1018/1480 train_time:155169ms step_avg:153.94ms
step:1019/1480 train_time:155329ms step_avg:153.94ms
step:1020/1480 train_time:155489ms step_avg:153.95ms
step:1021/1480 train_time:155648ms step_avg:153.95ms
step:1022/1480 train_time:155806ms step_avg:153.96ms
step:1023/1480 train_time:155967ms step_avg:153.97ms
step:1024/1480 train_time:156127ms step_avg:153.97ms
step:1025/1480 train_time:156289ms step_avg:153.98ms
step:1026/1480 train_time:156448ms step_avg:153.98ms
step:1027/1480 train_time:156606ms step_avg:153.99ms
step:1028/1480 train_time:156768ms step_avg:154.00ms
step:1029/1480 train_time:156931ms step_avg:154.00ms
step:1030/1480 train_time:157090ms step_avg:154.01ms
step:1031/1480 train_time:157250ms step_avg:154.02ms
step:1032/1480 train_time:157413ms step_avg:154.02ms
step:1033/1480 train_time:157572ms step_avg:154.03ms
step:1034/1480 train_time:157731ms step_avg:154.03ms
step:1035/1480 train_time:157891ms step_avg:154.04ms
step:1036/1480 train_time:158052ms step_avg:154.05ms
step:1037/1480 train_time:158212ms step_avg:154.05ms
step:1038/1480 train_time:158372ms step_avg:154.06ms
step:1039/1480 train_time:158535ms step_avg:154.07ms
step:1040/1480 train_time:158694ms step_avg:154.07ms
step:1041/1480 train_time:158857ms step_avg:154.08ms
step:1042/1480 train_time:159017ms step_avg:154.09ms
step:1043/1480 train_time:159175ms step_avg:154.09ms
step:1044/1480 train_time:159335ms step_avg:154.10ms
step:1045/1480 train_time:159496ms step_avg:154.10ms
step:1046/1480 train_time:159657ms step_avg:154.11ms
step:1047/1480 train_time:159816ms step_avg:154.11ms
step:1048/1480 train_time:159979ms step_avg:154.12ms
step:1049/1480 train_time:160140ms step_avg:154.13ms
step:1050/1480 train_time:160301ms step_avg:154.14ms
step:1051/1480 train_time:160465ms step_avg:154.14ms
step:1052/1480 train_time:160625ms step_avg:154.15ms
step:1053/1480 train_time:160787ms step_avg:154.16ms
step:1054/1480 train_time:160948ms step_avg:154.16ms
step:1055/1480 train_time:161107ms step_avg:154.17ms
step:1056/1480 train_time:161266ms step_avg:154.17ms
step:1057/1480 train_time:161424ms step_avg:154.18ms
step:1058/1480 train_time:161586ms step_avg:154.19ms
step:1059/1480 train_time:161749ms step_avg:154.19ms
step:1060/1480 train_time:161910ms step_avg:154.20ms
step:1061/1480 train_time:162068ms step_avg:154.20ms
step:1062/1480 train_time:162226ms step_avg:154.21ms
step:1063/1480 train_time:162386ms step_avg:154.21ms
step:1064/1480 train_time:162543ms step_avg:154.22ms
step:1065/1480 train_time:162704ms step_avg:154.22ms
step:1066/1480 train_time:162866ms step_avg:154.23ms
step:1067/1480 train_time:163026ms step_avg:154.24ms
step:1068/1480 train_time:163186ms step_avg:154.24ms
step:1069/1480 train_time:163349ms step_avg:154.25ms
step:1070/1480 train_time:163508ms step_avg:154.25ms
step:1071/1480 train_time:163672ms step_avg:154.26ms
step:1072/1480 train_time:163830ms step_avg:154.27ms
step:1073/1480 train_time:163989ms step_avg:154.27ms
step:1074/1480 train_time:164148ms step_avg:154.27ms
step:1075/1480 train_time:164308ms step_avg:154.28ms
step:1076/1480 train_time:164468ms step_avg:154.28ms
step:1077/1480 train_time:164625ms step_avg:154.29ms
step:1078/1480 train_time:164790ms step_avg:154.30ms
step:1079/1480 train_time:164953ms step_avg:154.31ms
step:1080/1480 train_time:165115ms step_avg:154.31ms
step:1081/1480 train_time:165275ms step_avg:154.32ms
step:1082/1480 train_time:165434ms step_avg:154.32ms
step:1083/1480 train_time:165593ms step_avg:154.33ms
step:1084/1480 train_time:165752ms step_avg:154.33ms
step:1085/1480 train_time:165912ms step_avg:154.34ms
step:1086/1480 train_time:166075ms step_avg:154.34ms
step:1087/1480 train_time:166235ms step_avg:154.35ms
step:1088/1480 train_time:166397ms step_avg:154.36ms
step:1089/1480 train_time:166562ms step_avg:154.37ms
step:1090/1480 train_time:166725ms step_avg:154.37ms
step:1091/1480 train_time:166885ms step_avg:154.38ms
step:1092/1480 train_time:167047ms step_avg:154.39ms
step:1093/1480 train_time:167208ms step_avg:154.39ms
step:1094/1480 train_time:167368ms step_avg:154.40ms
step:1095/1480 train_time:167526ms step_avg:154.40ms
step:1096/1480 train_time:167687ms step_avg:154.41ms
step:1097/1480 train_time:167850ms step_avg:154.42ms
step:1098/1480 train_time:168011ms step_avg:154.42ms
step:1099/1480 train_time:168173ms step_avg:154.43ms
step:1100/1480 train_time:168337ms step_avg:154.44ms
step:1101/1480 train_time:168500ms step_avg:154.45ms
step:1102/1480 train_time:168663ms step_avg:154.45ms
step:1103/1480 train_time:168829ms step_avg:154.46ms
step:1104/1480 train_time:168990ms step_avg:154.47ms
step:1105/1480 train_time:169152ms step_avg:154.48ms
step:1106/1480 train_time:169313ms step_avg:154.48ms
step:1107/1480 train_time:169473ms step_avg:154.49ms
step:1108/1480 train_time:169632ms step_avg:154.49ms
step:1109/1480 train_time:169790ms step_avg:154.50ms
step:1110/1480 train_time:169952ms step_avg:154.50ms
step:1111/1480 train_time:170115ms step_avg:154.51ms
step:1112/1480 train_time:170280ms step_avg:154.52ms
step:1113/1480 train_time:170450ms step_avg:154.53ms
step:1114/1480 train_time:170612ms step_avg:154.54ms
step:1115/1480 train_time:170773ms step_avg:154.55ms
step:1116/1480 train_time:170932ms step_avg:154.55ms
step:1117/1480 train_time:171097ms step_avg:154.56ms
step:1118/1480 train_time:171263ms step_avg:154.57ms
step:1119/1480 train_time:171424ms step_avg:154.57ms
step:1120/1480 train_time:171586ms step_avg:154.58ms
step:1121/1480 train_time:171748ms step_avg:154.59ms
step:1122/1480 train_time:171907ms step_avg:154.59ms
step:1123/1480 train_time:172067ms step_avg:154.60ms
step:1124/1480 train_time:172228ms step_avg:154.60ms
step:1125/1480 train_time:172389ms step_avg:154.61ms
step:1125/1480 val_loss:3.3827 train_time:172463ms step_avg:154.68ms
step:1126/1480 train_time:172558ms step_avg:154.62ms
step:1127/1480 train_time:172713ms step_avg:154.62ms
step:1128/1480 train_time:172873ms step_avg:154.63ms
step:1129/1480 train_time:173036ms step_avg:154.63ms
step:1130/1480 train_time:173197ms step_avg:154.64ms
step:1131/1480 train_time:173367ms step_avg:154.65ms
step:1132/1480 train_time:173526ms step_avg:154.66ms
step:1133/1480 train_time:173689ms step_avg:154.67ms
step:1134/1480 train_time:173851ms step_avg:154.67ms
step:1135/1480 train_time:174011ms step_avg:154.68ms
step:1136/1480 train_time:174174ms step_avg:154.68ms
step:1137/1480 train_time:174335ms step_avg:154.69ms
step:1138/1480 train_time:174503ms step_avg:154.70ms
step:1139/1480 train_time:174673ms step_avg:154.71ms
step:1140/1480 train_time:174826ms step_avg:154.71ms
step:1141/1480 train_time:174990ms step_avg:154.72ms
step:1142/1480 train_time:175150ms step_avg:154.73ms
step:1143/1480 train_time:175315ms step_avg:154.73ms
step:1144/1480 train_time:175478ms step_avg:154.74ms
step:1145/1480 train_time:175637ms step_avg:154.75ms
step:1146/1480 train_time:175802ms step_avg:154.76ms
step:1147/1480 train_time:175964ms step_avg:154.76ms
step:1148/1480 train_time:176125ms step_avg:154.77ms
step:1149/1480 train_time:176286ms step_avg:154.77ms
step:1150/1480 train_time:176447ms step_avg:154.78ms
step:1151/1480 train_time:176610ms step_avg:154.79ms
step:1152/1480 train_time:176774ms step_avg:154.79ms
step:1153/1480 train_time:176938ms step_avg:154.80ms
step:1154/1480 train_time:177099ms step_avg:154.81ms
step:1155/1480 train_time:177259ms step_avg:154.81ms
step:1156/1480 train_time:177428ms step_avg:154.82ms
step:1157/1480 train_time:177589ms step_avg:154.83ms
step:1158/1480 train_time:177750ms step_avg:154.83ms
step:1159/1480 train_time:177910ms step_avg:154.84ms
step:1160/1480 train_time:178069ms step_avg:154.84ms
step:1161/1480 train_time:178231ms step_avg:154.85ms
step:1162/1480 train_time:178393ms step_avg:154.85ms
step:1163/1480 train_time:178556ms step_avg:154.86ms
step:1164/1480 train_time:178720ms step_avg:154.87ms
step:1165/1480 train_time:178880ms step_avg:154.87ms
step:1166/1480 train_time:179043ms step_avg:154.88ms
step:1167/1480 train_time:179203ms step_avg:154.89ms
step:1168/1480 train_time:179365ms step_avg:154.89ms
step:1169/1480 train_time:179527ms step_avg:154.90ms
step:1170/1480 train_time:179687ms step_avg:154.90ms
step:1171/1480 train_time:179848ms step_avg:154.91ms
step:1172/1480 train_time:180007ms step_avg:154.91ms
step:1173/1480 train_time:180169ms step_avg:154.92ms
step:1174/1480 train_time:180341ms step_avg:154.93ms
step:1175/1480 train_time:180504ms step_avg:154.94ms
step:1176/1480 train_time:180666ms step_avg:154.95ms
step:1177/1480 train_time:180833ms step_avg:154.96ms
step:1178/1480 train_time:180993ms step_avg:154.96ms
step:1179/1480 train_time:181153ms step_avg:154.96ms
step:1180/1480 train_time:181325ms step_avg:154.98ms
step:1181/1480 train_time:181487ms step_avg:154.98ms
step:1182/1480 train_time:181649ms step_avg:154.99ms
step:1183/1480 train_time:181809ms step_avg:155.00ms
step:1184/1480 train_time:181971ms step_avg:155.00ms
step:1185/1480 train_time:182135ms step_avg:155.01ms
step:1186/1480 train_time:182299ms step_avg:155.02ms
step:1187/1480 train_time:182471ms step_avg:155.03ms
step:1188/1480 train_time:182630ms step_avg:155.03ms
step:1189/1480 train_time:182791ms step_avg:155.04ms
step:1190/1480 train_time:182951ms step_avg:155.04ms
step:1191/1480 train_time:183115ms step_avg:155.05ms
step:1192/1480 train_time:183276ms step_avg:155.06ms
step:1193/1480 train_time:183437ms step_avg:155.06ms
step:1194/1480 train_time:183599ms step_avg:155.07ms
step:1195/1480 train_time:183761ms step_avg:155.07ms
step:1196/1480 train_time:183933ms step_avg:155.09ms
step:1197/1480 train_time:184093ms step_avg:155.09ms
step:1198/1480 train_time:184262ms step_avg:155.10ms
step:1199/1480 train_time:184426ms step_avg:155.11ms
step:1200/1480 train_time:184587ms step_avg:155.12ms
step:1201/1480 train_time:184748ms step_avg:155.12ms
step:1202/1480 train_time:184917ms step_avg:155.13ms
step:1203/1480 train_time:185083ms step_avg:155.14ms
step:1204/1480 train_time:185249ms step_avg:155.15ms
step:1205/1480 train_time:185409ms step_avg:155.15ms
step:1206/1480 train_time:185569ms step_avg:155.16ms
step:1207/1480 train_time:185730ms step_avg:155.16ms
step:1208/1480 train_time:185890ms step_avg:155.17ms
step:1209/1480 train_time:186052ms step_avg:155.17ms
step:1210/1480 train_time:186219ms step_avg:155.18ms
step:1211/1480 train_time:186382ms step_avg:155.19ms
step:1212/1480 train_time:186546ms step_avg:155.20ms
step:1213/1480 train_time:186710ms step_avg:155.20ms
step:1214/1480 train_time:186875ms step_avg:155.21ms
step:1215/1480 train_time:187040ms step_avg:155.22ms
step:1216/1480 train_time:187202ms step_avg:155.23ms
step:1217/1480 train_time:187364ms step_avg:155.23ms
step:1218/1480 train_time:187527ms step_avg:155.24ms
step:1219/1480 train_time:187693ms step_avg:155.25ms
step:1220/1480 train_time:187856ms step_avg:155.25ms
step:1221/1480 train_time:188017ms step_avg:155.26ms
step:1222/1480 train_time:188177ms step_avg:155.26ms
step:1223/1480 train_time:188342ms step_avg:155.27ms
step:1224/1480 train_time:188509ms step_avg:155.28ms
step:1225/1480 train_time:188672ms step_avg:155.29ms
step:1226/1480 train_time:188838ms step_avg:155.29ms
step:1227/1480 train_time:189003ms step_avg:155.30ms
step:1228/1480 train_time:189166ms step_avg:155.31ms
step:1229/1480 train_time:189328ms step_avg:155.31ms
step:1230/1480 train_time:189496ms step_avg:155.32ms
step:1231/1480 train_time:189662ms step_avg:155.33ms
step:1232/1480 train_time:189828ms step_avg:155.34ms
step:1233/1480 train_time:189988ms step_avg:155.35ms
step:1234/1480 train_time:190149ms step_avg:155.35ms
step:1235/1480 train_time:190313ms step_avg:155.36ms
step:1236/1480 train_time:190474ms step_avg:155.36ms
step:1237/1480 train_time:190636ms step_avg:155.37ms
step:1238/1480 train_time:190810ms step_avg:155.38ms
step:1239/1480 train_time:190972ms step_avg:155.39ms
step:1240/1480 train_time:191137ms step_avg:155.40ms
step:1241/1480 train_time:191304ms step_avg:155.41ms
step:1242/1480 train_time:191465ms step_avg:155.41ms
step:1243/1480 train_time:191630ms step_avg:155.42ms
step:1244/1480 train_time:191789ms step_avg:155.42ms
step:1245/1480 train_time:191952ms step_avg:155.43ms
step:1246/1480 train_time:192114ms step_avg:155.43ms
step:1247/1480 train_time:192279ms step_avg:155.44ms
step:1248/1480 train_time:192442ms step_avg:155.45ms
step:1249/1480 train_time:192605ms step_avg:155.45ms
step:1250/1480 train_time:192767ms step_avg:155.46ms
step:1250/1480 val_loss:3.3334 train_time:192843ms step_avg:155.52ms
step:1251/1480 train_time:192938ms step_avg:155.47ms
step:1252/1480 train_time:193101ms step_avg:155.48ms
step:1253/1480 train_time:193261ms step_avg:155.48ms
step:1254/1480 train_time:193422ms step_avg:155.48ms
step:1255/1480 train_time:193593ms step_avg:155.50ms
step:1256/1480 train_time:193760ms step_avg:155.51ms
step:1257/1480 train_time:193922ms step_avg:155.51ms
step:1258/1480 train_time:194087ms step_avg:155.52ms
step:1259/1480 train_time:194250ms step_avg:155.52ms
step:1260/1480 train_time:194409ms step_avg:155.53ms
step:1261/1480 train_time:194572ms step_avg:155.53ms
step:1262/1480 train_time:194737ms step_avg:155.54ms
step:1263/1480 train_time:194905ms step_avg:155.55ms
step:1264/1480 train_time:195064ms step_avg:155.55ms
step:1265/1480 train_time:195223ms step_avg:155.56ms
step:1266/1480 train_time:195386ms step_avg:155.56ms
step:1267/1480 train_time:195546ms step_avg:155.57ms
step:1268/1480 train_time:195709ms step_avg:155.57ms
step:1269/1480 train_time:195876ms step_avg:155.58ms
step:1270/1480 train_time:196039ms step_avg:155.59ms
step:1271/1480 train_time:196203ms step_avg:155.59ms
step:1272/1480 train_time:196362ms step_avg:155.60ms
step:1273/1480 train_time:196525ms step_avg:155.60ms
step:1274/1480 train_time:196689ms step_avg:155.61ms
step:1275/1480 train_time:196850ms step_avg:155.61ms
step:1276/1480 train_time:197011ms step_avg:155.62ms
step:1277/1480 train_time:197175ms step_avg:155.62ms
step:1278/1480 train_time:197337ms step_avg:155.63ms
step:1279/1480 train_time:197500ms step_avg:155.63ms
step:1280/1480 train_time:197665ms step_avg:155.64ms
step:1281/1480 train_time:197826ms step_avg:155.65ms
step:1282/1480 train_time:197985ms step_avg:155.65ms
step:1283/1480 train_time:198147ms step_avg:155.65ms
step:1284/1480 train_time:198310ms step_avg:155.66ms
step:1285/1480 train_time:198472ms step_avg:155.66ms
step:1286/1480 train_time:198634ms step_avg:155.67ms
step:1287/1480 train_time:198797ms step_avg:155.67ms
step:1288/1480 train_time:198960ms step_avg:155.68ms
step:1289/1480 train_time:199129ms step_avg:155.69ms
step:1290/1480 train_time:199298ms step_avg:155.70ms
step:1291/1480 train_time:199463ms step_avg:155.71ms
step:1292/1480 train_time:199626ms step_avg:155.71ms
step:1293/1480 train_time:199794ms step_avg:155.72ms
step:1294/1480 train_time:199958ms step_avg:155.73ms
step:1295/1480 train_time:200122ms step_avg:155.74ms
step:1296/1480 train_time:200284ms step_avg:155.74ms
step:1297/1480 train_time:200446ms step_avg:155.75ms
step:1298/1480 train_time:200608ms step_avg:155.75ms
step:1299/1480 train_time:200770ms step_avg:155.76ms
step:1300/1480 train_time:200930ms step_avg:155.76ms
step:1301/1480 train_time:201093ms step_avg:155.77ms
step:1302/1480 train_time:201259ms step_avg:155.77ms
step:1303/1480 train_time:201425ms step_avg:155.78ms
step:1304/1480 train_time:201589ms step_avg:155.79ms
step:1305/1480 train_time:201750ms step_avg:155.79ms
step:1306/1480 train_time:201915ms step_avg:155.80ms
step:1307/1480 train_time:202077ms step_avg:155.80ms
step:1308/1480 train_time:202239ms step_avg:155.81ms
step:1309/1480 train_time:202404ms step_avg:155.82ms
step:1310/1480 train_time:202566ms step_avg:155.82ms
step:1311/1480 train_time:202727ms step_avg:155.82ms
step:1312/1480 train_time:202893ms step_avg:155.83ms
step:1313/1480 train_time:203056ms step_avg:155.84ms
step:1314/1480 train_time:203220ms step_avg:155.84ms
step:1315/1480 train_time:203383ms step_avg:155.85ms
step:1316/1480 train_time:203542ms step_avg:155.85ms
step:1317/1480 train_time:203704ms step_avg:155.86ms
step:1318/1480 train_time:203871ms step_avg:155.86ms
step:1319/1480 train_time:204038ms step_avg:155.87ms
step:1320/1480 train_time:204206ms step_avg:155.88ms
step:1321/1480 train_time:204368ms step_avg:155.89ms
step:1322/1480 train_time:204540ms step_avg:155.90ms
step:1323/1480 train_time:204704ms step_avg:155.91ms
step:1324/1480 train_time:204867ms step_avg:155.91ms
step:1325/1480 train_time:205036ms step_avg:155.92ms
step:1326/1480 train_time:205204ms step_avg:155.93ms
step:1327/1480 train_time:205365ms step_avg:155.93ms
step:1328/1480 train_time:205527ms step_avg:155.94ms
step:1329/1480 train_time:205713ms step_avg:155.96ms
step:1330/1480 train_time:205877ms step_avg:155.97ms
step:1331/1480 train_time:206040ms step_avg:155.97ms
step:1332/1480 train_time:206205ms step_avg:155.98ms
step:1333/1480 train_time:206370ms step_avg:155.99ms
step:1334/1480 train_time:206531ms step_avg:155.99ms
step:1335/1480 train_time:206692ms step_avg:155.99ms
step:1336/1480 train_time:206862ms step_avg:156.00ms
step:1337/1480 train_time:207026ms step_avg:156.01ms
step:1338/1480 train_time:207190ms step_avg:156.02ms
step:1339/1480 train_time:207354ms step_avg:156.02ms
step:1340/1480 train_time:207517ms step_avg:156.03ms
step:1341/1480 train_time:207681ms step_avg:156.03ms
step:1342/1480 train_time:207846ms step_avg:156.04ms
step:1343/1480 train_time:208007ms step_avg:156.04ms
step:1344/1480 train_time:208168ms step_avg:156.05ms
step:1345/1480 train_time:208337ms step_avg:156.06ms
step:1346/1480 train_time:208500ms step_avg:156.06ms
step:1347/1480 train_time:208663ms step_avg:156.07ms
step:1348/1480 train_time:208825ms step_avg:156.07ms
step:1349/1480 train_time:208986ms step_avg:156.08ms
step:1350/1480 train_time:209152ms step_avg:156.08ms
step:1351/1480 train_time:209315ms step_avg:156.09ms
step:1352/1480 train_time:209478ms step_avg:156.09ms
step:1353/1480 train_time:209643ms step_avg:156.10ms
step:1354/1480 train_time:209806ms step_avg:156.11ms
step:1355/1480 train_time:209966ms step_avg:156.11ms
step:1356/1480 train_time:210130ms step_avg:156.11ms
step:1357/1480 train_time:210294ms step_avg:156.12ms
step:1358/1480 train_time:210459ms step_avg:156.13ms
step:1359/1480 train_time:210624ms step_avg:156.13ms
step:1360/1480 train_time:210789ms step_avg:156.14ms
step:1361/1480 train_time:210957ms step_avg:156.15ms
step:1362/1480 train_time:211123ms step_avg:156.16ms
step:1363/1480 train_time:211291ms step_avg:156.17ms
step:1364/1480 train_time:211454ms step_avg:156.17ms
step:1365/1480 train_time:211613ms step_avg:156.17ms
step:1366/1480 train_time:211776ms step_avg:156.18ms
step:1367/1480 train_time:211939ms step_avg:156.18ms
step:1368/1480 train_time:212105ms step_avg:156.19ms
step:1369/1480 train_time:212275ms step_avg:156.20ms
step:1370/1480 train_time:212440ms step_avg:156.21ms
step:1371/1480 train_time:212605ms step_avg:156.21ms
step:1372/1480 train_time:212772ms step_avg:156.22ms
step:1373/1480 train_time:212934ms step_avg:156.22ms
step:1374/1480 train_time:213101ms step_avg:156.23ms
step:1375/1480 train_time:213264ms step_avg:156.24ms
step:1375/1480 val_loss:3.2946 train_time:213338ms step_avg:156.29ms
step:1376/1480 train_time:213430ms step_avg:156.24ms
step:1377/1480 train_time:213591ms step_avg:156.25ms
step:1378/1480 train_time:213753ms step_avg:156.25ms
step:1379/1480 train_time:213919ms step_avg:156.26ms
step:1380/1480 train_time:214083ms step_avg:156.27ms
step:1381/1480 train_time:214251ms step_avg:156.27ms
step:1382/1480 train_time:214414ms step_avg:156.28ms
step:1383/1480 train_time:214577ms step_avg:156.28ms
step:1384/1480 train_time:214744ms step_avg:156.29ms
step:1385/1480 train_time:214905ms step_avg:156.29ms
step:1386/1480 train_time:215068ms step_avg:156.30ms
step:1387/1480 train_time:215231ms step_avg:156.30ms
step:1388/1480 train_time:215391ms step_avg:156.31ms
step:1389/1480 train_time:215555ms step_avg:156.31ms
step:1390/1480 train_time:215717ms step_avg:156.32ms
step:1391/1480 train_time:215883ms step_avg:156.32ms
step:1392/1480 train_time:216049ms step_avg:156.33ms
step:1393/1480 train_time:216211ms step_avg:156.33ms
step:1394/1480 train_time:216373ms step_avg:156.34ms
step:1395/1480 train_time:216534ms step_avg:156.34ms
step:1396/1480 train_time:216695ms step_avg:156.35ms
step:1397/1480 train_time:216855ms step_avg:156.35ms
step:1398/1480 train_time:217015ms step_avg:156.35ms
step:1399/1480 train_time:217177ms step_avg:156.36ms
step:1400/1480 train_time:217349ms step_avg:156.37ms
step:1401/1480 train_time:217509ms step_avg:156.37ms
step:1402/1480 train_time:217672ms step_avg:156.37ms
step:1403/1480 train_time:217837ms step_avg:156.38ms
step:1404/1480 train_time:218000ms step_avg:156.38ms
step:1405/1480 train_time:218166ms step_avg:156.39ms
step:1406/1480 train_time:218331ms step_avg:156.40ms
step:1407/1480 train_time:218491ms step_avg:156.40ms
step:1408/1480 train_time:218652ms step_avg:156.40ms
step:1409/1480 train_time:218825ms step_avg:156.42ms
step:1410/1480 train_time:218987ms step_avg:156.42ms
step:1411/1480 train_time:219148ms step_avg:156.42ms
step:1412/1480 train_time:219310ms step_avg:156.43ms
step:1413/1480 train_time:219472ms step_avg:156.43ms
step:1414/1480 train_time:219636ms step_avg:156.44ms
step:1415/1480 train_time:219800ms step_avg:156.44ms
step:1416/1480 train_time:219976ms step_avg:156.45ms
step:1417/1480 train_time:220142ms step_avg:156.46ms
step:1418/1480 train_time:220307ms step_avg:156.47ms
step:1419/1480 train_time:220472ms step_avg:156.47ms
step:1420/1480 train_time:220637ms step_avg:156.48ms
step:1421/1480 train_time:220802ms step_avg:156.49ms
step:1422/1480 train_time:220968ms step_avg:156.49ms
step:1423/1480 train_time:221130ms step_avg:156.50ms
step:1424/1480 train_time:221295ms step_avg:156.50ms
step:1425/1480 train_time:221466ms step_avg:156.51ms
step:1426/1480 train_time:221630ms step_avg:156.52ms
step:1427/1480 train_time:221795ms step_avg:156.52ms
step:1428/1480 train_time:221956ms step_avg:156.53ms
step:1429/1480 train_time:222117ms step_avg:156.53ms
step:1430/1480 train_time:222281ms step_avg:156.54ms
step:1431/1480 train_time:222449ms step_avg:156.54ms
step:1432/1480 train_time:222616ms step_avg:156.55ms
step:1433/1480 train_time:222785ms step_avg:156.56ms
step:1434/1480 train_time:222955ms step_avg:156.57ms
step:1435/1480 train_time:223120ms step_avg:156.58ms
step:1436/1480 train_time:223286ms step_avg:156.58ms
step:1437/1480 train_time:223449ms step_avg:156.59ms
step:1438/1480 train_time:223611ms step_avg:156.59ms
step:1439/1480 train_time:223776ms step_avg:156.60ms
step:1440/1480 train_time:223940ms step_avg:156.60ms
step:1441/1480 train_time:224105ms step_avg:156.61ms
step:1442/1480 train_time:224272ms step_avg:156.61ms
step:1443/1480 train_time:224445ms step_avg:156.63ms
step:1444/1480 train_time:224609ms step_avg:156.63ms
step:1445/1480 train_time:224771ms step_avg:156.63ms
step:1446/1480 train_time:224936ms step_avg:156.64ms
step:1447/1480 train_time:225105ms step_avg:156.65ms
step:1448/1480 train_time:225268ms step_avg:156.65ms
step:1449/1480 train_time:225432ms step_avg:156.66ms
step:1450/1480 train_time:225596ms step_avg:156.66ms
step:1451/1480 train_time:225759ms step_avg:156.67ms
step:1452/1480 train_time:225926ms step_avg:156.68ms
step:1453/1480 train_time:226089ms step_avg:156.68ms
step:1454/1480 train_time:226252ms step_avg:156.68ms
step:1455/1480 train_time:226418ms step_avg:156.69ms
step:1456/1480 train_time:226582ms step_avg:156.70ms
step:1457/1480 train_time:226745ms step_avg:156.70ms
step:1458/1480 train_time:226907ms step_avg:156.70ms
step:1459/1480 train_time:227073ms step_avg:156.71ms
step:1460/1480 train_time:227236ms step_avg:156.71ms
step:1461/1480 train_time:227400ms step_avg:156.72ms
step:1462/1480 train_time:227565ms step_avg:156.73ms
step:1463/1480 train_time:227731ms step_avg:156.73ms
step:1464/1480 train_time:227896ms step_avg:156.74ms
step:1465/1480 train_time:228060ms step_avg:156.74ms
step:1466/1480 train_time:228222ms step_avg:156.75ms
step:1467/1480 train_time:228388ms step_avg:156.75ms
step:1468/1480 train_time:228551ms step_avg:156.76ms
step:1469/1480 train_time:228714ms step_avg:156.76ms
step:1470/1480 train_time:228881ms step_avg:156.77ms
step:1471/1480 train_time:229052ms step_avg:156.78ms
step:1472/1480 train_time:229222ms step_avg:156.79ms
step:1473/1480 train_time:229386ms step_avg:156.79ms
step:1474/1480 train_time:229552ms step_avg:156.80ms
step:1475/1480 train_time:229723ms step_avg:156.81ms
step:1476/1480 train_time:229887ms step_avg:156.81ms
step:1477/1480 train_time:230055ms step_avg:156.82ms
step:1478/1480 train_time:230226ms step_avg:156.83ms
step:1479/1480 train_time:230392ms step_avg:156.84ms
step:1480/1480 train_time:230554ms step_avg:156.84ms
step:1480/1480 val_loss:3.2758 train_time:230630ms step_avg:156.89ms
peak memory consumption: 34239 MiB