train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing sagemaker will throw a ContextualVersionConflict due to site-packages containing\n",
    "# two versions of urllib3. Even though the right urllib3 version (1.23) is active, we need to\n",
    "# manually remove urllib3 1.25.3 & restart kernel to make sagemaker work.\n",
    "# This could probably be moved to post_setup.sh, but leaving it here in case someone knows\n",
    "# of a solution\n",
    "\n",
    "import urllib3\n",
    "assert(urllib3.__version__ == '1.23')\n",
    "\n",
    "# !sudo rm -rf /opt/miniconda/envs/py3/lib/python3.6/site-packages/urllib3-1.25.3.dist-info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import datetime\n",
    "\n",
    "import boto3\n",
    "import sagemaker\n",
    "from sagemaker.tensorflow import TensorFlow\n",
    "import tensorflow\n",
    "\n",
    "from config import DATASET_URL, IMAGE_NAME, IAM_ROLE, MODEL_URL_PREFIX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_sample_count = 60000  # Standard size of training data for MNIST\n",
    "batch_size = 64\n",
    "num_epochs = 10\n",
    "\n",
    "training_step_count = int(training_sample_count * num_epochs / batch_size)\n",
    "\n",
    "entry_point = 'model.py'\n",
    "timestamp = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')\n",
    "output_path   = f\"{MODEL_URL_PREFIX}{timestamp}/output\"\n",
    "code_location = f\"{MODEL_URL_PREFIX}{timestamp}/code\"\n",
    "\n",
    "boto_session = boto3.Session(region_name=\"us-east-1\",\n",
    "                             aws_access_key_id=os.environ.get(\"AWS_ACCESS_KEY_ID_PROD\"),\n",
    "                             aws_secret_access_key=os.environ.get(\"AWS_SECRET_ACCESS_KEY_PROD\"))\n",
    "\n",
    "kwargs = dict(entry_point=entry_point,\n",
    "              image_name=IMAGE_NAME,\n",
    "              role=IAM_ROLE,\n",
    "              sagemaker_session=sagemaker.Session(boto_session=boto_session),\n",
    "              train_instance_count=1,\n",
    "              train_instance_type='ml.m5.xlarge',\n",
    "              framework_version='1.13',\n",
    "              hyperparameters={'dataset-url': DATASET_URL,\n",
    "                               'training_steps': training_step_count,\n",
    "                               'batch_size': batch_size,\n",
    "                               'evaluation_steps': 10,},\n",
    "              py_version = 'py3',\n",
    "              output_path = output_path,\n",
    "              code_location=code_location,\n",
    "              distributions={'parameter_server': {'enabled': True}})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on a single sagemaker instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-02 18:43:48 Starting - Starting the training job...\n",
      "2019-07-02 18:43:50 Starting - Launching requested ML instances......\n",
      "2019-07-02 18:44:50 Starting - Preparing the instances for training...\n",
      "2019-07-02 18:45:49 Downloading - Downloading input data\n",
      "2019-07-02 18:45:49 Training - Downloading the training image......\n",
      "2019-07-02 18:46:47 Training - Training image download completed. Training in progress..\n",
      "\u001b[31m2019-07-02 18:46:50,617 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:50,623 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:50,966 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:50,981 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:50,991 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[31mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[31m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-43-48-315\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-43-48-315/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[31m}\n",
      "\u001b[0m\n",
      "\u001b[31mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[31mSM_HOSTS=[\"algo-1\"]\u001b[0m\n",
      "\u001b[31mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[31mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[31mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[31mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[31mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[31mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[31mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[31mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[31mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[31mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[31mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[31mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[31mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[31mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[31mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-43-48-315/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[31mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-43-48-315\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-43-48-315/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[31mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[31mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[31mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[31mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[31mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model\u001b[0m\n",
      "\u001b[31mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[31mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[31mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[31m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[31maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:52,550 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': allow_soft_placement: true\u001b[0m\n",
      "\u001b[31mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[31m}\u001b[0m\n",
      "\u001b[31m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f4c8f891c18>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\u001b[0m\n",
      "\u001b[31mInside training script on worker with (0-based) index 0 out of 0.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:52,976 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:52,977 - tensorflow - INFO - Running training and evaluation locally (non-distributed).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:52,979 - tensorflow - INFO - Start train and evaluate loop. The evaluate will happen after every checkpoint. Checkpoint frequency is determined based on RunConfig arguments: save_checkpoints_steps 500 or save_checkpoints_secs None.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:53.069602: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:46:53.070265: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:47:01,323 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[31mInstructions for updating:\u001b[0m\n",
      "\u001b[31mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01,343 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01,804 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01,806 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.842278: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.842335: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.880407: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.880461: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.900579: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:01.900616: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.088180: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.088226: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.138277: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.138321: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02,594 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.608864: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.608899: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02,720 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02,726 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.778688: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:02.778737: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:03,238 - tensorflow - INFO - Saving checkpoints for 0 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:04.521310: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:04.521347: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:04,865 - tensorflow - INFO - loss = 87.202965, step = 1\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:07,712 - tensorflow - INFO - global_step/sec: 35.1168\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:07,714 - tensorflow - INFO - loss = 2.03685, step = 101 (2.849 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:11,449 - tensorflow - INFO - global_step/sec: 26.7615\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:11,451 - tensorflow - INFO - loss = 1.7797443, step = 201 (3.738 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:15,286 - tensorflow - INFO - global_step/sec: 26.0615\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:15,287 - tensorflow - INFO - loss = 1.1699357, step = 301 (3.836 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:18,463 - tensorflow - INFO - global_step/sec: 31.4772\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:18,464 - tensorflow - INFO - loss = 0.99785733, step = 401 (3.177 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:22,117 - tensorflow - INFO - Saving checkpoints for 500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:23.440767: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:23.440810: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:23,824 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:23,969 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:23,993 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:47:23\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,093 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,156 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,335 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,341 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,404 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,413 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,425 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,439 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,451 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,464 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,478 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,491 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,504 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,518 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,534 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:47:24\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24,534 - tensorflow - INFO - Saving dict for global step 500: accuracy = 0.7609375, global_step = 500, loss = 0.76433235\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.546458: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.546496: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.584982: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.585017: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.601200: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:24.601235: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:25,072 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:25,188 - tensorflow - INFO - global_step/sec: 14.8693\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:25,189 - tensorflow - INFO - loss = 0.90263647, step = 501 (6.725 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:28,030 - tensorflow - INFO - global_step/sec: 35.1863\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:28,031 - tensorflow - INFO - loss = 0.52383846, step = 601 (2.842 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:31,692 - tensorflow - INFO - global_step/sec: 27.3122\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:31,693 - tensorflow - INFO - loss = 0.44048715, step = 701 (3.662 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:47:35,264 - tensorflow - INFO - global_step/sec: 27.9899\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:35,266 - tensorflow - INFO - loss = 0.7298758, step = 801 (3.573 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:38,687 - tensorflow - INFO - global_step/sec: 29.2218\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:38,688 - tensorflow - INFO - loss = 0.9928016, step = 901 (3.422 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:42,760 - tensorflow - INFO - Saving checkpoints for 1000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:43.947977: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:43.948022: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,386 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,537 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,559 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:47:44\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,740 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,802 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-1000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,978 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:44,984 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,035 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,047 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,060 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,073 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,086 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,098 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,118 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,134 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,151 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,169 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,191 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:47:45\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,191 - tensorflow - INFO - Saving dict for global step 1000: accuracy = 0.928125, global_step = 1000, loss = 0.32401612\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,247 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 1000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-1000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,350 - tensorflow - INFO - global_step/sec: 15.0061\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:45,352 - tensorflow - INFO - loss = 0.37389216, step = 1001 (6.664 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:48,229 - tensorflow - INFO - global_step/sec: 34.7396\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:48,230 - tensorflow - INFO - loss = 0.5550946, step = 1101 (2.879 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:51,911 - tensorflow - INFO - global_step/sec: 27.1593\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:51,912 - tensorflow - INFO - loss = 0.277103, step = 1201 (3.682 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:55,151 - tensorflow - INFO - global_step/sec: 30.8668\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:55,153 - tensorflow - INFO - loss = 0.4287386, step = 1301 (3.241 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:58,579 - tensorflow - INFO - global_step/sec: 29.1665\u001b[0m\n",
      "\u001b[31m2019-07-02 18:47:58,581 - tensorflow - INFO - loss = 0.40374422, step = 1401 (3.428 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:02,165 - tensorflow - INFO - Saving checkpoints for 1500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:03.484582: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:03.484625: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,122 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,278 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,301 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:48:04\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,401 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,471 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-1500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,653 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,659 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,710 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,723 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,736 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,748 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,761 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,774 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,787 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,799 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,813 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,825 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,840 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:48:04\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,840 - tensorflow - INFO - Saving dict for global step 1500: accuracy = 0.8640625, global_step = 1500, loss = 0.5218694\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,895 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 1500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-1500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,987 - tensorflow - INFO - global_step/sec: 15.606\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:04,988 - tensorflow - INFO - loss = 0.32266653, step = 1501 (6.407 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:08,153 - tensorflow - INFO - global_step/sec: 31.5835\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:08,155 - tensorflow - INFO - loss = 0.282652, step = 1601 (3.167 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:12,283 - tensorflow - INFO - global_step/sec: 24.2175\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:12,284 - tensorflow - INFO - loss = 0.5056141, step = 1701 (4.129 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:16,034 - tensorflow - INFO - global_step/sec: 26.6568\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:16,035 - tensorflow - INFO - loss = 0.4267323, step = 1801 (3.752 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:20,649 - tensorflow - INFO - global_step/sec: 21.6684\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:20,650 - tensorflow - INFO - loss = 0.15916735, step = 1901 (4.615 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:24,555 - tensorflow - INFO - Saving checkpoints for 2000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:25.560978: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:25.561024: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,162 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,310 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,334 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:48:26\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,433 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,503 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-2000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,694 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,699 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,749 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,763 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,782 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:26,805 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,106 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,121 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,134 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,147 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,160 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,172 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,189 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:48:27\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,190 - tensorflow - INFO - Saving dict for global step 2000: accuracy = 0.921875, global_step = 2000, loss = 0.28365773\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,299 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 2000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-2000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,414 - tensorflow - INFO - global_step/sec: 14.7819\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:27,415 - tensorflow - INFO - loss = 0.35900632, step = 2001 (6.765 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:29,979 - tensorflow - INFO - global_step/sec: 38.9917\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:29,980 - tensorflow - INFO - loss = 0.326855, step = 2101 (2.565 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:33,692 - tensorflow - INFO - global_step/sec: 26.9332\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:33,698 - tensorflow - INFO - loss = 0.45200408, step = 2201 (3.718 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:36,902 - tensorflow - INFO - global_step/sec: 31.1534\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:36,902 - tensorflow - INFO - loss = 0.36539024, step = 2301 (3.204 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:48:40,210 - tensorflow - INFO - global_step/sec: 30.2274\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:40,211 - tensorflow - INFO - loss = 0.9276402, step = 2401 (3.309 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:44,324 - tensorflow - INFO - Saving checkpoints for 2500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:45.664410: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:45.664449: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:45,971 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,122 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,146 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:48:46\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,247 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,306 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-2500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,470 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,476 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,527 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,540 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,553 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,566 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,579 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,592 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,604 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,616 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,630 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,648 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,664 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:48:46\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,665 - tensorflow - INFO - Saving dict for global step 2500: accuracy = 0.9765625, global_step = 2500, loss = 0.11363838\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,835 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 2500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-2500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,968 - tensorflow - INFO - global_step/sec: 14.7965\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:46,969 - tensorflow - INFO - loss = 0.43392608, step = 2501 (6.758 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:50,241 - tensorflow - INFO - global_step/sec: 30.555\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:50,242 - tensorflow - INFO - loss = 0.29549208, step = 2601 (3.273 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:53,426 - tensorflow - INFO - global_step/sec: 31.3928\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:53,427 - tensorflow - INFO - loss = 0.33523166, step = 2701 (3.186 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:56,892 - tensorflow - INFO - global_step/sec: 28.8553\u001b[0m\n",
      "\u001b[31m2019-07-02 18:48:56,894 - tensorflow - INFO - loss = 0.10501526, step = 2801 (3.466 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:00,388 - tensorflow - INFO - global_step/sec: 28.6055\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:00,389 - tensorflow - INFO - loss = 0.24266256, step = 2901 (3.495 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:03,796 - tensorflow - INFO - Saving checkpoints for 3000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:05.374043: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:05.374084: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:05,956 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,111 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,134 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:49:06\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,329 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,391 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-3000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,571 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,577 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,629 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,642 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,655 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,669 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,682 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,697 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,711 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,725 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,738 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,751 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,767 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:49:06\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,767 - tensorflow - INFO - Saving dict for global step 3000: accuracy = 0.965625, global_step = 3000, loss = 0.12927838\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,842 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 3000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-3000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,898 - tensorflow - INFO - global_step/sec: 15.3603\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:06,899 - tensorflow - INFO - loss = 0.35513848, step = 3001 (6.510 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:09,829 - tensorflow - INFO - global_step/sec: 34.1228\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:09,830 - tensorflow - INFO - loss = 0.40684557, step = 3101 (2.931 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:13,476 - tensorflow - INFO - global_step/sec: 27.4136\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:13,477 - tensorflow - INFO - loss = 0.36087728, step = 3201 (3.647 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:16,728 - tensorflow - INFO - global_step/sec: 30.7597\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:16,729 - tensorflow - INFO - loss = 0.39890814, step = 3301 (3.251 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:21,020 - tensorflow - INFO - global_step/sec: 23.2941\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:21,021 - tensorflow - INFO - loss = 0.24331903, step = 3401 (4.293 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:25,731 - tensorflow - INFO - Saving checkpoints for 3500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:26.975550: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:26.975589: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,360 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,508 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,530 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:49:27\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,629 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,686 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-3500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,843 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,848 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,899 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,912 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,925 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,938 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,957 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,971 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,986 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:27,999 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,017 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,036 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,055 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:49:28\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,056 - tensorflow - INFO - Saving dict for global step 3500: accuracy = 0.9375, global_step = 3500, loss = 0.22441213\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,270 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 3500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-3500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,460 - tensorflow - INFO - global_step/sec: 13.4411\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:28,462 - tensorflow - INFO - loss = 0.13059755, step = 3501 (7.440 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:31,690 - tensorflow - INFO - global_step/sec: 30.9594\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:31,692 - tensorflow - INFO - loss = 0.025559073, step = 3601 (3.230 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:35,864 - tensorflow - INFO - global_step/sec: 23.9609\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:35,865 - tensorflow - INFO - loss = 0.21321158, step = 3701 (4.174 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:49:39,601 - tensorflow - INFO - global_step/sec: 26.7599\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:39,602 - tensorflow - INFO - loss = 0.31026232, step = 3801 (3.737 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:42,407 - tensorflow - INFO - global_step/sec: 35.6401\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:42,408 - tensorflow - INFO - loss = 0.085260645, step = 3901 (2.806 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:45,947 - tensorflow - INFO - Saving checkpoints for 4000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:47.524771: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:47.524809: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:47,912 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,062 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,085 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:49:48\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,188 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,257 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-4000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,412 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,418 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,469 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,482 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,494 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,508 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,519 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,531 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,545 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,557 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,570 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,583 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,598 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:49:48\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,599 - tensorflow - INFO - Saving dict for global step 4000: accuracy = 0.9625, global_step = 4000, loss = 0.1407496\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,692 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 4000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-4000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,762 - tensorflow - INFO - global_step/sec: 15.7346\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:48,763 - tensorflow - INFO - loss = 0.15088154, step = 4001 (6.355 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:51,961 - tensorflow - INFO - global_step/sec: 31.2549\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:51,963 - tensorflow - INFO - loss = 0.2189039, step = 4101 (3.200 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:55,890 - tensorflow - INFO - global_step/sec: 25.4564\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:55,891 - tensorflow - INFO - loss = 0.25083867, step = 4201 (3.928 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:59,916 - tensorflow - INFO - global_step/sec: 24.8373\u001b[0m\n",
      "\u001b[31m2019-07-02 18:49:59,917 - tensorflow - INFO - loss = 0.064477004, step = 4301 (4.026 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:03,321 - tensorflow - INFO - global_step/sec: 29.3666\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:03,322 - tensorflow - INFO - loss = 0.27689973, step = 4401 (3.405 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:07,003 - tensorflow - INFO - Saving checkpoints for 4500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08.299978: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08.300028: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08,663 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08,811 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08,835 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:50:08\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:08,932 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,020 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-4500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,195 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,201 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,260 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,276 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,298 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,314 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,579 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,593 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,605 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,617 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,631 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,644 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,660 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:50:09\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,660 - tensorflow - INFO - Saving dict for global step 4500: accuracy = 0.9796875, global_step = 4500, loss = 0.057816155\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,756 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 4500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-4500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,840 - tensorflow - INFO - global_step/sec: 15.3412\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:09,841 - tensorflow - INFO - loss = 0.086208954, step = 4501 (6.518 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:12,838 - tensorflow - INFO - global_step/sec: 33.3493\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:12,840 - tensorflow - INFO - loss = 0.4365328, step = 4601 (2.999 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:16,102 - tensorflow - INFO - global_step/sec: 30.636\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:16,104 - tensorflow - INFO - loss = 0.34644568, step = 4701 (3.264 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:20,079 - tensorflow - INFO - global_step/sec: 25.1467\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:20,080 - tensorflow - INFO - loss = 0.11381133, step = 4801 (3.977 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:24,050 - tensorflow - INFO - global_step/sec: 25.1819\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:24,051 - tensorflow - INFO - loss = 0.2504616, step = 4901 (3.971 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:27,564 - tensorflow - INFO - Saving checkpoints for 5000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:28.844989: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:28.845056: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,244 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,394 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,417 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:50:29\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,605 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,669 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-5000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,918 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,923 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,975 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:29,989 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,002 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,016 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,034 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,058 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,074 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,094 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,110 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,131 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,150 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:50:30\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,150 - tensorflow - INFO - Saving dict for global step 5000: accuracy = 0.9828125, global_step = 5000, loss = 0.053162385\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,234 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 5000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-5000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,438 - tensorflow - INFO - global_step/sec: 15.6534\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:30,440 - tensorflow - INFO - loss = 0.11119475, step = 5001 (6.389 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:33,853 - tensorflow - INFO - global_step/sec: 29.2832\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:33,855 - tensorflow - INFO - loss = 0.17594674, step = 5101 (3.415 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:37,354 - tensorflow - INFO - global_step/sec: 28.566\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:37,355 - tensorflow - INFO - loss = 0.03821931, step = 5201 (3.500 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:50:40,765 - tensorflow - INFO - global_step/sec: 29.3148\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:40,767 - tensorflow - INFO - loss = 0.10550964, step = 5301 (3.412 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:43,876 - tensorflow - INFO - global_step/sec: 32.1487\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:43,877 - tensorflow - INFO - loss = 0.29030272, step = 5401 (3.110 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:47,098 - tensorflow - INFO - Saving checkpoints for 5500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48.286933: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48.286976: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48,691 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48,839 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48,861 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:50:48\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:48,960 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,025 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-5500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,189 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,194 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,244 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,257 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,270 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,282 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,295 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,308 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,319 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,332 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,344 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,357 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,372 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:50:49\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,372 - tensorflow - INFO - Saving dict for global step 5500: accuracy = 0.98125, global_step = 5500, loss = 0.051914223\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,469 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 5500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-5500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,610 - tensorflow - INFO - global_step/sec: 17.4409\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:49,615 - tensorflow - INFO - loss = 0.3848521, step = 5501 (5.738 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:52,608 - tensorflow - INFO - global_step/sec: 33.3459\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:52,610 - tensorflow - INFO - loss = 0.048525922, step = 5601 (2.995 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:56,080 - tensorflow - INFO - global_step/sec: 28.806\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:56,081 - tensorflow - INFO - loss = 0.13264537, step = 5701 (3.472 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:59,788 - tensorflow - INFO - global_step/sec: 26.9699\u001b[0m\n",
      "\u001b[31m2019-07-02 18:50:59,789 - tensorflow - INFO - loss = 0.20679973, step = 5801 (3.707 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:02,992 - tensorflow - INFO - global_step/sec: 31.2099\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:02,993 - tensorflow - INFO - loss = 0.11506958, step = 5901 (3.204 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:07,179 - tensorflow - INFO - Saving checkpoints for 6000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:08.792098: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:08.792136: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:09,624 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:09,768 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:09,880 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:51:09\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:09,979 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,050 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-6000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,208 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,213 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,263 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,277 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,295 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,310 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,325 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,346 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,364 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,616 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,628 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,641 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,656 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:51:10\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,657 - tensorflow - INFO - Saving dict for global step 6000: accuracy = 0.9875, global_step = 6000, loss = 0.048066925\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,724 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 6000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-6000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,823 - tensorflow - INFO - global_step/sec: 12.7694\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:10,824 - tensorflow - INFO - loss = 0.16543296, step = 6001 (7.831 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:13,740 - tensorflow - INFO - global_step/sec: 34.2794\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:13,741 - tensorflow - INFO - loss = 0.2575702, step = 6101 (2.917 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:18,618 - tensorflow - INFO - global_step/sec: 20.5024\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:18,619 - tensorflow - INFO - loss = 0.28826416, step = 6201 (4.877 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:21,928 - tensorflow - INFO - global_step/sec: 30.2102\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:21,929 - tensorflow - INFO - loss = 0.069808096, step = 6301 (3.311 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:25,658 - tensorflow - INFO - global_step/sec: 26.8099\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:25,659 - tensorflow - INFO - loss = 0.4440125, step = 6401 (3.730 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:28,942 - tensorflow - INFO - Saving checkpoints for 6500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:30.679811: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:30.679848: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,054 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,199 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,222 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:51:31\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,320 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,381 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-6500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,563 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,569 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,619 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,632 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,644 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,657 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,669 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,682 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,702 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,718 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,739 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,757 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,776 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:51:31\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,776 - tensorflow - INFO - Saving dict for global step 6500: accuracy = 0.9390625, global_step = 6500, loss = 0.2077121\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,882 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 6500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-6500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,941 - tensorflow - INFO - global_step/sec: 15.9162\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:31,942 - tensorflow - INFO - loss = 0.058733236, step = 6501 (6.283 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:35,604 - tensorflow - INFO - global_step/sec: 27.2967\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:35,605 - tensorflow - INFO - loss = 0.22147354, step = 6601 (3.663 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:39,787 - tensorflow - INFO - global_step/sec: 23.9063\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:39,788 - tensorflow - INFO - loss = 0.31067544, step = 6701 (4.183 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:51:42,942 - tensorflow - INFO - global_step/sec: 31.698\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:42,944 - tensorflow - INFO - loss = 0.08908516, step = 6801 (3.156 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:46,743 - tensorflow - INFO - global_step/sec: 26.3095\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:46,744 - tensorflow - INFO - loss = 0.046961665, step = 6901 (3.800 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:50,072 - tensorflow - INFO - Saving checkpoints for 7000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:51.615627: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:51.615665: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,303 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,543 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,566 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:51:52\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,664 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,734 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-7000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,942 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,947 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:52,998 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,011 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,025 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,037 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,050 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,064 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,077 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,089 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,101 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,114 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,131 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:51:53\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,131 - tensorflow - INFO - Saving dict for global step 7000: accuracy = 0.953125, global_step = 7000, loss = 0.16844584\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,220 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 7000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-7000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,299 - tensorflow - INFO - global_step/sec: 15.2517\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:53,301 - tensorflow - INFO - loss = 0.10036361, step = 7001 (6.557 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:56,085 - tensorflow - INFO - global_step/sec: 35.897\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:56,086 - tensorflow - INFO - loss = 0.047650192, step = 7101 (2.786 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:59,725 - tensorflow - INFO - global_step/sec: 27.473\u001b[0m\n",
      "\u001b[31m2019-07-02 18:51:59,727 - tensorflow - INFO - loss = 0.19648224, step = 7201 (3.641 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:03,440 - tensorflow - INFO - global_step/sec: 26.9194\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:03,441 - tensorflow - INFO - loss = 0.17197493, step = 7301 (3.714 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:07,047 - tensorflow - INFO - global_step/sec: 27.7242\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:07,048 - tensorflow - INFO - loss = 0.082084, step = 7401 (3.607 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:10,300 - tensorflow - INFO - Saving checkpoints for 7500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:11.769632: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:11.769673: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,178 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,325 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,348 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:52:12\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,447 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,507 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-7500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,754 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,759 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,810 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,823 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,843 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,859 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,880 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,902 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:12,914 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,169 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,182 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,194 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,215 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:52:13\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,216 - tensorflow - INFO - Saving dict for global step 7500: accuracy = 0.96875, global_step = 7500, loss = 0.13576396\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,266 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 7500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-7500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,376 - tensorflow - INFO - global_step/sec: 15.8005\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:13,377 - tensorflow - INFO - loss = 0.21195753, step = 7501 (6.329 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:16,292 - tensorflow - INFO - global_step/sec: 34.2928\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:16,293 - tensorflow - INFO - loss = 0.15196513, step = 7601 (2.917 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:19,662 - tensorflow - INFO - global_step/sec: 29.6688\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:19,664 - tensorflow - INFO - loss = 0.10523659, step = 7701 (3.371 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:23,258 - tensorflow - INFO - global_step/sec: 27.8144\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:23,259 - tensorflow - INFO - loss = 0.01366056, step = 7801 (3.595 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:26,854 - tensorflow - INFO - global_step/sec: 27.8082\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:26,855 - tensorflow - INFO - loss = 0.1000916, step = 7901 (3.596 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:30,296 - tensorflow - INFO - Saving checkpoints for 8000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:31.696193: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:31.696232: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,075 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,317 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,340 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:52:32\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,443 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,501 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-8000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,659 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,665 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,717 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,730 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,743 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,756 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,769 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,782 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,795 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,808 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,826 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,839 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,853 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:52:32\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,854 - tensorflow - INFO - Saving dict for global step 8000: accuracy = 0.965625, global_step = 8000, loss = 0.12229706\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:32,949 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 8000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-8000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:33,009 - tensorflow - INFO - global_step/sec: 16.2476\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:33,010 - tensorflow - INFO - loss = 0.08642995, step = 8001 (6.155 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:36,142 - tensorflow - INFO - global_step/sec: 31.9129\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:36,144 - tensorflow - INFO - loss = 0.34062916, step = 8101 (3.134 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:39,810 - tensorflow - INFO - global_step/sec: 27.2626\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:39,811 - tensorflow - INFO - loss = 0.34421217, step = 8201 (3.668 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:52:43,101 - tensorflow - INFO - global_step/sec: 30.3914\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:43,102 - tensorflow - INFO - loss = 0.019097788, step = 8301 (3.291 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:46,673 - tensorflow - INFO - global_step/sec: 27.9919\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:46,674 - tensorflow - INFO - loss = 0.15632853, step = 8401 (3.572 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:50,323 - tensorflow - INFO - Saving checkpoints for 8500 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:51.755041: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:51.755078: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,223 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,369 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,392 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:52:52\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,492 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,551 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-8500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,720 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,726 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,775 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,789 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,801 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,813 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,826 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,839 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,852 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,864 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,877 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,890 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,907 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:52:52\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,907 - tensorflow - INFO - Saving dict for global step 8500: accuracy = 0.9703125, global_step = 8500, loss = 0.06591032\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:52,982 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 8500: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-8500\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:53,099 - tensorflow - INFO - global_step/sec: 15.5626\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:53,100 - tensorflow - INFO - loss = 0.13656205, step = 8501 (6.426 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:56,220 - tensorflow - INFO - global_step/sec: 32.0417\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:56,221 - tensorflow - INFO - loss = 0.1068071, step = 8601 (3.121 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:59,198 - tensorflow - INFO - global_step/sec: 33.5788\u001b[0m\n",
      "\u001b[31m2019-07-02 18:52:59,199 - tensorflow - INFO - loss = 0.12705228, step = 8701 (2.978 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:02,811 - tensorflow - INFO - global_step/sec: 27.6775\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:02,812 - tensorflow - INFO - loss = 0.10623005, step = 8801 (3.613 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:06,402 - tensorflow - INFO - global_step/sec: 27.8435\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:06,403 - tensorflow - INFO - loss = 0.24040174, step = 8901 (3.592 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:09,791 - tensorflow - INFO - Saving checkpoints for 9000 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:11.330505: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:11.330544: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:12,036 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:12,186 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:12,209 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:53:12\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:12,308 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:12,599 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-9000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,074 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,080 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,132 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,146 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,159 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,172 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,192 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,204 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,217 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,234 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,246 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,462 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,478 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:53:13\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:13,478 - tensorflow - INFO - Saving dict for global step 9000: accuracy = 0.9640625, global_step = 9000, loss = 0.13518196\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:14,116 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 9000: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-9000\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:14,188 - tensorflow - INFO - global_step/sec: 12.8434\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:14,189 - tensorflow - INFO - loss = 0.4716417, step = 9001 (7.786 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:17,308 - tensorflow - INFO - global_step/sec: 32.054\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:17,309 - tensorflow - INFO - loss = 0.20660187, step = 9101 (3.120 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:20,673 - tensorflow - INFO - global_step/sec: 29.7173\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:20,674 - tensorflow - INFO - loss = 0.3414672, step = 9201 (3.365 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:24,447 - tensorflow - INFO - global_step/sec: 26.4982\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:24,448 - tensorflow - INFO - loss = 0.1032249, step = 9301 (3.774 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:27,130 - tensorflow - INFO - Saving checkpoints for 9375 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:28.842279: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:28.842377: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,395 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,543 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,567 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:53:29\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,668 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,730 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-9375\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,896 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,902 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,952 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,966 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,978 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:29,991 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,004 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,017 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,030 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,043 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,057 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,070 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,087 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:53:30\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,087 - tensorflow - INFO - Saving dict for global step 9375: accuracy = 0.965625, global_step = 9375, loss = 0.14460222\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,167 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 9375: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-43-48-315/model/model.ckpt-9375\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,215 - tensorflow - DEBUG - Calling exporter with the `is_the_final_export=True`.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,345 - tensorflow - INFO - Loss for final step: 0.12066976.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,746 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[31mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[31mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[31m2019-07-02 18:53:30,747 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2019-07-02 18:53:37 Uploading - Uploading generated training model\n",
      "2019-07-02 18:53:37 Completed - Training job completed\n",
      "Billable seconds: 470\n",
      "CPU times: user 1.24 s, sys: 132 ms, total: 1.37 s\n",
      "Wall time: 10min 17s\n"
     ]
    }
   ],
   "source": [
    "mnist_estimator = TensorFlow(**kwargs)\n",
    "%time  mnist_estimator.fit(inputs=None)  # we're bypassing the conventional sagemaker input methods"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Accuracy should be around 0.95 and wall time = 10min 17s"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on 5 sagemaker instances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With just minimal code changes, we can the same number of steps in total but distributed over 5 instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2019-07-02 18:54:06 Starting - Starting the training job...\n",
      "2019-07-02 18:54:07 Starting - Launching requested ML instances......\n",
      "2019-07-02 18:55:08 Starting - Preparing the instances for training......\n",
      "2019-07-02 18:56:12 Downloading - Downloading input data\n",
      "2019-07-02 18:56:12 Training - Downloading the training image....\n",
      "\u001b[31m2019-07-02 18:57:09,640 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,646 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,807 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,807 sagemaker_tensorflow_container.training INFO     Launching parameter server process\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,807 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,851 sagemaker_tensorflow_container.training INFO     Launching worker process\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,957 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,972 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:09,982 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[31mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[31m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-3\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\",\n",
      "        \"algo-3\",\n",
      "        \"algo-4\",\n",
      "        \"algo-5\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": false,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-54-05-832\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-3\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\",\n",
      "            \"algo-3\",\n",
      "            \"algo-4\",\n",
      "            \"algo-5\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[31m}\n",
      "\u001b[0m\n",
      "\u001b[31mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[31mSM_HOSTS=[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"]\u001b[0m\n",
      "\u001b[31mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[31mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[31mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[31mSM_RESOURCE_CONFIG={\"current_host\":\"algo-3\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[31mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[31mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[31mSM_CURRENT_HOST=algo-3\u001b[0m\n",
      "\u001b[31mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[31mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[31mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[31mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[31mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[31mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[31mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[31mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[31mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-3\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":false,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-54-05-832\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-3\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[31mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[31mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[31mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[31mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[31mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\u001b[0m\n",
      "\u001b[31mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[31mTF_CONFIG={\"cluster\": {\"master\": [\"algo-1:2222\"], \"ps\": [\"algo-1:2223\", \"algo-2:2223\", \"algo-3:2223\", \"algo-4:2223\", \"algo-5:2223\"], \"worker\": [\"algo-2:2222\", \"algo-3:2222\", \"algo-4:2222\", \"algo-5:2222\"]}, \"environment\": \"cloud\", \"task\": {\"index\": 1, \"type\": \"worker\"}}\u001b[0m\n",
      "\u001b[31mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[31mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[31m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,315 - tensorflow - INFO - TF_CONFIG environment variable: {'cluster': {'master': ['algo-1:2222'], 'ps': ['algo-1:2223', 'algo-2:2223', 'algo-3:2223', 'algo-4:2223', 'algo-5:2223'], 'worker': ['algo-2:2222', 'algo-3:2222', 'algo-4:2222', 'algo-5:2222']}, 'environment': 'cloud', 'task': {'index': 1, 'type': 'worker'}}\u001b[0m\n",
      "\u001b[31mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[31maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,315 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_filters: \"/job:ps\"\u001b[0m\n",
      "\u001b[31mdevice_filters: \"/job:worker/task:1\"\u001b[0m\n",
      "\u001b[31mallow_soft_placement: true\u001b[0m\n",
      "\u001b[31mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[31m}\u001b[0m\n",
      "\u001b[31m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f43cbc11be0>, '_task_type': 'worker', '_task_id': 1, '_global_id_in_cluster': 2, '_master': 'grpc://algo-3:2222', '_evaluation_master': '', '_num_ps_replicas': 5, '_num_worker_replicas': 5, '_is_chief': False}\u001b[0m\n",
      "\u001b[31mInside training script on worker with (0-based) index 2 out of 4.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,658 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,663 - tensorflow - INFO - Start Tensorflow server.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,749 - tensorflow - INFO - Waiting 10 secs before starting training.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31m2019-07-02 18:57:10,680 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,686 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,836 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,837 sagemaker_tensorflow_container.training INFO     Launching parameter server process\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,837 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,879 sagemaker_tensorflow_container.training INFO     Launching worker process\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:10,998 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,016 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:11,026 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[31mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[31m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-1\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\",\n",
      "        \"algo-3\",\n",
      "        \"algo-4\",\n",
      "        \"algo-5\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": true,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-54-05-832\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-1\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\",\n",
      "            \"algo-3\",\n",
      "            \"algo-4\",\n",
      "            \"algo-5\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[31m}\n",
      "\u001b[0m\n",
      "\u001b[31mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[31mSM_HOSTS=[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"]\u001b[0m\n",
      "\u001b[31mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[31mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[31mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[31mSM_RESOURCE_CONFIG={\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[31mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[31mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[31mSM_CURRENT_HOST=algo-1\u001b[0m\n",
      "\u001b[31mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[31mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[31mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[31mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[31mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[31mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[31mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[31mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[31mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[31mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-1\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":true,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-54-05-832\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-1\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[31mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[31mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[31mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[31mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[31mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[31mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\u001b[0m\n",
      "\u001b[31mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[31mTF_CONFIG={\"cluster\": {\"master\": [\"algo-1:2222\"], \"ps\": [\"algo-1:2223\", \"algo-2:2223\", \"algo-3:2223\", \"algo-4:2223\", \"algo-5:2223\"], \"worker\": [\"algo-2:2222\", \"algo-3:2222\", \"algo-4:2222\", \"algo-5:2222\"]}, \"environment\": \"cloud\", \"task\": {\"index\": 0, \"type\": \"master\"}}\u001b[0m\n",
      "\u001b[31mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[31mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[31m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:10,967 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:10,973 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,126 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,127 sagemaker_tensorflow_container.training INFO     Launching parameter server process\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,127 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,176 sagemaker_tensorflow_container.training INFO     Launching worker process\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,280 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,295 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:11,306 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[35mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[35m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-5\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\",\n",
      "        \"algo-3\",\n",
      "        \"algo-4\",\n",
      "        \"algo-5\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": false,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-54-05-832\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-5\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\",\n",
      "            \"algo-3\",\n",
      "            \"algo-4\",\n",
      "            \"algo-5\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[35m}\n",
      "\u001b[0m\n",
      "\u001b[35mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[35mSM_HOSTS=[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"]\u001b[0m\n",
      "\u001b[35mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[35mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[35mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[35mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[35mSM_RESOURCE_CONFIG={\"current_host\":\"algo-5\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[35mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[35mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[35mSM_CURRENT_HOST=algo-5\u001b[0m\n",
      "\u001b[35mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[35mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[35mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[35mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[35mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[35mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[35mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[35mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[35mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[35mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-5\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":false,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-54-05-832\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-5\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[35mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[35mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[35mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[35mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[35mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[35mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\u001b[0m\n",
      "\u001b[35mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[35mTF_CONFIG={\"cluster\": {\"master\": [\"algo-1:2222\"], \"ps\": [\"algo-1:2223\", \"algo-2:2223\", \"algo-3:2223\", \"algo-4:2223\", \"algo-5:2223\"], \"worker\": [\"algo-2:2222\", \"algo-3:2222\", \"algo-4:2222\", \"algo-5:2222\"]}, \"environment\": \"cloud\", \"task\": {\"index\": 3, \"type\": \"worker\"}}\u001b[0m\n",
      "\u001b[35mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[35mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[35m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,557 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,563 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,721 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,721 sagemaker_tensorflow_container.training INFO     Launching parameter server process\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,721 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,766 sagemaker_tensorflow_container.training INFO     Launching worker process\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,914 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,929 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:09,940 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[32mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[32m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-2\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\",\n",
      "        \"algo-3\",\n",
      "        \"algo-4\",\n",
      "        \"algo-5\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": false,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-54-05-832\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-2\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\",\n",
      "            \"algo-3\",\n",
      "            \"algo-4\",\n",
      "            \"algo-5\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[32m}\n",
      "\u001b[0m\n",
      "\u001b[32mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[32mSM_HOSTS=[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"]\u001b[0m\n",
      "\u001b[32mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[32mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[32mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[32mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[32mSM_RESOURCE_CONFIG={\"current_host\":\"algo-2\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[32mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[32mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[32mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[32mSM_CURRENT_HOST=algo-2\u001b[0m\n",
      "\u001b[32mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[32mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[32mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[32mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[32mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[32mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[32mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[32mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[32mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[32mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[32mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-2\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":false,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-54-05-832\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-2\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[32mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[32mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[32mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[32mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[32mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[32mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\u001b[0m\n",
      "\u001b[32mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[32mTF_CONFIG={\"cluster\": {\"master\": [\"algo-1:2222\"], \"ps\": [\"algo-1:2223\", \"algo-2:2223\", \"algo-3:2223\", \"algo-4:2223\", \"algo-5:2223\"], \"worker\": [\"algo-2:2222\", \"algo-3:2222\", \"algo-4:2222\", \"algo-5:2222\"]}, \"environment\": \"cloud\", \"task\": {\"index\": 0, \"type\": \"worker\"}}\u001b[0m\n",
      "\u001b[32mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[32mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[32m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:11,270 - tensorflow - INFO - TF_CONFIG environment variable: {'cluster': {'master': ['algo-1:2222'], 'ps': ['algo-1:2223', 'algo-2:2223', 'algo-3:2223', 'algo-4:2223', 'algo-5:2223'], 'worker': ['algo-2:2222', 'algo-3:2222', 'algo-4:2222', 'algo-5:2222']}, 'environment': 'cloud', 'task': {'index': 0, 'type': 'worker'}}\u001b[0m\n",
      "\u001b[32mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[32maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:11,270 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_filters: \"/job:ps\"\u001b[0m\n",
      "\u001b[32mdevice_filters: \"/job:worker/task:0\"\u001b[0m\n",
      "\u001b[32mallow_soft_placement: true\u001b[0m\n",
      "\u001b[32mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[32m}\u001b[0m\n",
      "\u001b[32m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fdfc4f12b70>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 1, '_master': 'grpc://algo-2:2222', '_evaluation_master': '', '_num_ps_replicas': 5, '_num_worker_replicas': 5, '_is_chief': False}\u001b[0m\n",
      "\u001b[32mInside training script on worker with (0-based) index 1 out of 4.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:11,680 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:11,685 - tensorflow - INFO - Start Tensorflow server.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:11,748 - tensorflow - INFO - Waiting 5 secs before starting training.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:12,513 - tensorflow - INFO - TF_CONFIG environment variable: {'cluster': {'master': ['algo-1:2222'], 'ps': ['algo-1:2223', 'algo-2:2223', 'algo-3:2223', 'algo-4:2223', 'algo-5:2223'], 'worker': ['algo-2:2222', 'algo-3:2222', 'algo-4:2222', 'algo-5:2222']}, 'environment': 'cloud', 'task': {'index': 0, 'type': 'master'}}\u001b[0m\n",
      "\u001b[31mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[31maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:12,514 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_filters: \"/job:ps\"\u001b[0m\n",
      "\u001b[31mdevice_filters: \"/job:master\"\u001b[0m\n",
      "\u001b[31mallow_soft_placement: true\u001b[0m\n",
      "\u001b[31mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[31m}\u001b[0m\n",
      "\u001b[31m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7faf68f51be0>, '_task_type': 'master', '_task_id': 0, '_global_id_in_cluster': 0, '_master': 'grpc://algo-1:2222', '_evaluation_master': '', '_num_ps_replicas': 5, '_num_worker_replicas': 5, '_is_chief': True}\u001b[0m\n",
      "\u001b[31mInside training script on worker with (0-based) index 0 out of 4.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:12,859 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:12,861 - tensorflow - INFO - Start Tensorflow server.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:13.030926: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:13.031030: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:12,719 - tensorflow - INFO - TF_CONFIG environment variable: {'cluster': {'master': ['algo-1:2222'], 'ps': ['algo-1:2223', 'algo-2:2223', 'algo-3:2223', 'algo-4:2223', 'algo-5:2223'], 'worker': ['algo-2:2222', 'algo-3:2222', 'algo-4:2222', 'algo-5:2222']}, 'environment': 'cloud', 'task': {'index': 3, 'type': 'worker'}}\u001b[0m\n",
      "\u001b[35mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[35maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:12,720 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_filters: \"/job:ps\"\u001b[0m\n",
      "\u001b[35mdevice_filters: \"/job:worker/task:3\"\u001b[0m\n",
      "\u001b[35mallow_soft_placement: true\u001b[0m\n",
      "\u001b[35mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[35m}\u001b[0m\n",
      "\u001b[35m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fe0999d1be0>, '_task_type': 'worker', '_task_id': 3, '_global_id_in_cluster': 4, '_master': 'grpc://algo-5:2222', '_evaluation_master': '', '_num_ps_replicas': 5, '_num_worker_replicas': 5, '_is_chief': False}\u001b[0m\n",
      "\u001b[35mInside training script on worker with (0-based) index 4 out of 4.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:13,021 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:13,032 - tensorflow - INFO - Start Tensorflow server.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:13,111 - tensorflow - INFO - Waiting 20 secs before starting training.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,273 sagemaker-containers INFO     Imported framework sagemaker_tensorflow_container.training\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,278 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,541 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,541 sagemaker_tensorflow_container.training INFO     Launching parameter server process\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,541 sagemaker_tensorflow_container.training INFO     Running distributed training job with parameter servers\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,590 sagemaker_tensorflow_container.training INFO     Launching worker process\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,710 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,726 sagemaker-containers INFO     No GPUs detected (normal if no gpus installed)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:12,736 sagemaker-containers INFO     Invoking user script\n",
      "\u001b[0m\n",
      "\u001b[34mTraining Env:\n",
      "\u001b[0m\n",
      "\u001b[34m{\n",
      "    \"additional_framework_parameters\": {\n",
      "        \"sagemaker_parameter_server_enabled\": true\n",
      "    },\n",
      "    \"channel_input_dirs\": {},\n",
      "    \"current_host\": \"algo-4\",\n",
      "    \"framework_module\": \"sagemaker_tensorflow_container.training:main\",\n",
      "    \"hosts\": [\n",
      "        \"algo-1\",\n",
      "        \"algo-2\",\n",
      "        \"algo-3\",\n",
      "        \"algo-4\",\n",
      "        \"algo-5\"\n",
      "    ],\n",
      "    \"hyperparameters\": {\n",
      "        \"batch_size\": 64,\n",
      "        \"evaluation_steps\": 10,\n",
      "        \"dataset-url\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\n",
      "        \"model_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\n",
      "        \"training_steps\": 9375\n",
      "    },\n",
      "    \"input_config_dir\": \"/opt/ml/input/config\",\n",
      "    \"input_data_config\": {},\n",
      "    \"input_dir\": \"/opt/ml/input\",\n",
      "    \"is_master\": false,\n",
      "    \"job_name\": \"petastorm-sagemaker-2019-07-02-18-54-05-832\",\n",
      "    \"log_level\": 20,\n",
      "    \"master_hostname\": \"algo-1\",\n",
      "    \"model_dir\": \"/opt/ml/model\",\n",
      "    \"module_dir\": \"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\n",
      "    \"module_name\": \"model\",\n",
      "    \"network_interface_name\": \"eth0\",\n",
      "    \"num_cpus\": 4,\n",
      "    \"num_gpus\": 0,\n",
      "    \"output_data_dir\": \"/opt/ml/output/data\",\n",
      "    \"output_dir\": \"/opt/ml/output\",\n",
      "    \"output_intermediate_dir\": \"/opt/ml/output/intermediate\",\n",
      "    \"resource_config\": {\n",
      "        \"current_host\": \"algo-4\",\n",
      "        \"hosts\": [\n",
      "            \"algo-1\",\n",
      "            \"algo-2\",\n",
      "            \"algo-3\",\n",
      "            \"algo-4\",\n",
      "            \"algo-5\"\n",
      "        ],\n",
      "        \"network_interface_name\": \"eth0\"\n",
      "    },\n",
      "    \"user_entry_point\": \"model.py\"\u001b[0m\n",
      "\u001b[34m}\n",
      "\u001b[0m\n",
      "\u001b[34mEnvironment variables:\n",
      "\u001b[0m\n",
      "\u001b[34mSM_HOSTS=[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"]\u001b[0m\n",
      "\u001b[34mSM_NETWORK_INTERFACE_NAME=eth0\u001b[0m\n",
      "\u001b[34mSM_HPS={\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375}\u001b[0m\n",
      "\u001b[34mSM_USER_ENTRY_POINT=model.py\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_PARAMS={\"sagemaker_parameter_server_enabled\":true}\u001b[0m\n",
      "\u001b[34mSM_RESOURCE_CONFIG={\"current_host\":\"algo-4\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"}\u001b[0m\n",
      "\u001b[34mSM_INPUT_DATA_CONFIG={}\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DATA_DIR=/opt/ml/output/data\u001b[0m\n",
      "\u001b[34mSM_CHANNELS=[]\u001b[0m\n",
      "\u001b[34mSM_CURRENT_HOST=algo-4\u001b[0m\n",
      "\u001b[34mSM_MODULE_NAME=model\u001b[0m\n",
      "\u001b[34mSM_LOG_LEVEL=20\u001b[0m\n",
      "\u001b[34mSM_FRAMEWORK_MODULE=sagemaker_tensorflow_container.training:main\u001b[0m\n",
      "\u001b[34mSM_INPUT_DIR=/opt/ml/input\u001b[0m\n",
      "\u001b[34mSM_INPUT_CONFIG_DIR=/opt/ml/input/config\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_DIR=/opt/ml/output\u001b[0m\n",
      "\u001b[34mSM_NUM_CPUS=4\u001b[0m\n",
      "\u001b[34mSM_NUM_GPUS=0\u001b[0m\n",
      "\u001b[34mSM_MODEL_DIR=/opt/ml/model\u001b[0m\n",
      "\u001b[34mSM_MODULE_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\u001b[0m\n",
      "\u001b[34mSM_TRAINING_ENV={\"additional_framework_parameters\":{\"sagemaker_parameter_server_enabled\":true},\"channel_input_dirs\":{},\"current_host\":\"algo-4\",\"framework_module\":\"sagemaker_tensorflow_container.training:main\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"hyperparameters\":{\"batch_size\":64,\"dataset-url\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"evaluation_steps\":10,\"model_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"training_steps\":9375},\"input_config_dir\":\"/opt/ml/input/config\",\"input_data_config\":{},\"input_dir\":\"/opt/ml/input\",\"is_master\":false,\"job_name\":\"petastorm-sagemaker-2019-07-02-18-54-05-832\",\"log_level\":20,\"master_hostname\":\"algo-1\",\"model_dir\":\"/opt/ml/model\",\"module_dir\":\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/code/petastorm-sagemaker-2019-07-02-18-54-05-832/source/sourcedir.tar.gz\",\"module_name\":\"model\",\"network_interface_name\":\"eth0\",\"num_cpus\":4,\"num_gpus\":0,\"output_data_dir\":\"/opt/ml/output/data\",\"output_dir\":\"/opt/ml/output\",\"output_intermediate_dir\":\"/opt/ml/output/intermediate\",\"resource_config\":{\"current_host\":\"algo-4\",\"hosts\":[\"algo-1\",\"algo-2\",\"algo-3\",\"algo-4\",\"algo-5\"],\"network_interface_name\":\"eth0\"},\"user_entry_point\":\"model.py\"}\u001b[0m\n",
      "\u001b[34mSM_USER_ARGS=[\"--batch_size\",\"64\",\"--dataset-url\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\",\"--evaluation_steps\",\"10\",\"--model_dir\",\"s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\",\"--training_steps\",\"9375\"]\u001b[0m\n",
      "\u001b[34mSM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate\u001b[0m\n",
      "\u001b[34mSM_HP_BATCH_SIZE=64\u001b[0m\n",
      "\u001b[34mSM_HP_EVALUATION_STEPS=10\u001b[0m\n",
      "\u001b[34mSM_HP_DATASET-URL=s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/\u001b[0m\n",
      "\u001b[34mSM_HP_MODEL_DIR=s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model\u001b[0m\n",
      "\u001b[34mSM_HP_TRAINING_STEPS=9375\u001b[0m\n",
      "\u001b[34mTF_CONFIG={\"cluster\": {\"master\": [\"algo-1:2222\"], \"ps\": [\"algo-1:2223\", \"algo-2:2223\", \"algo-3:2223\", \"algo-4:2223\", \"algo-5:2223\"], \"worker\": [\"algo-2:2222\", \"algo-3:2222\", \"algo-4:2222\", \"algo-5:2222\"]}, \"environment\": \"cloud\", \"task\": {\"index\": 2, \"type\": \"worker\"}}\u001b[0m\n",
      "\u001b[34mPYTHONPATH=/opt/ml/code:/usr/local/bin:/usr/lib/python36.zip:/usr/lib/python3.6:/usr/lib/python3.6/lib-dynload:/usr/local/lib/python3.6/dist-packages:/usr/lib/python3/dist-packages\n",
      "\u001b[0m\n",
      "\u001b[34mInvoking script with the following command:\n",
      "\u001b[0m\n",
      "\u001b[34m/usr/bin/python model.py --batch_size 64 --dataset-url s3://com.climate.production.analytics/dsw/scratch/sagemaker/sdegryze/mnist/ --evaluation_steps 10 --model_dir s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model --training_steps 9375\n",
      "\n",
      "\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:14,268 - tensorflow - INFO - TF_CONFIG environment variable: {'cluster': {'master': ['algo-1:2222'], 'ps': ['algo-1:2223', 'algo-2:2223', 'algo-3:2223', 'algo-4:2223', 'algo-5:2223'], 'worker': ['algo-2:2222', 'algo-3:2222', 'algo-4:2222', 'algo-5:2222']}, 'environment': 'cloud', 'task': {'index': 2, 'type': 'worker'}}\u001b[0m\n",
      "\u001b[34mLaunch tensorboard by running the following in terminal:\u001b[0m\n",
      "\u001b[34maws s3 sync {model_dir_parent_path} ~/Downloads/{model_dir_parent} && tensorboard --logdir=~/Downloads/{model_dir_parent}\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:14,269 - tensorflow - INFO - Using config: {'_model_dir': 's3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model', '_tf_random_seed': None, '_save_summary_steps': 50, '_save_checkpoints_steps': 500, '_save_checkpoints_secs': None, '_session_config': device_filters: \"/job:ps\"\u001b[0m\n",
      "\u001b[34mdevice_filters: \"/job:worker/task:2\"\u001b[0m\n",
      "\u001b[34mallow_soft_placement: true\u001b[0m\n",
      "\u001b[34mgraph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\u001b[0m\n",
      "\u001b[34m}\u001b[0m\n",
      "\u001b[34m, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7fc243511be0>, '_task_type': 'worker', '_task_id': 2, '_global_id_in_cluster': 3, '_master': 'grpc://algo-4:2222', '_evaluation_master': '', '_num_ps_replicas': 5, '_num_worker_replicas': 5, '_is_chief': False}\u001b[0m\n",
      "\u001b[34mInside training script on worker with (0-based) index 3 out of 4.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:14,580 - tensorflow - INFO - Not using Distribute Coordinator.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:14,582 - tensorflow - INFO - Start Tensorflow server.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:14,614 - tensorflow - INFO - Waiting 15 secs before starting training.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:16.828492: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:16.828531: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:21.863902: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:21.863943: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:21,919 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[32mInstructions for updating:\u001b[0m\n",
      "\u001b[32mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22,246 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[31mInstructions for updating:\u001b[0m\n",
      "\u001b[31mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22,269 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:21,940 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22,477 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22,479 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.488001: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.488038: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.505536: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.505576: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.545123: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.545161: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.561561: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:22.561598: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22,895 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22,897 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22.925822: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22.925867: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22.964704: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:22.964741: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23,450 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:23,088 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:23,203 - tensorflow - INFO - Waiting for model to be ready.  Ready_for_local_init_op:  Variables not initialized: global_step, dense/kernel, dense/bias, dense_1/kernel, dense_1/bias, dense_2/kernel, dense_2/bias, dense_3/kernel, dense_3/bias, dense_4/kernel, dense_4/bias, dense_5/kernel, dense_5/bias, beta1_power, beta2_power, dense/kernel/Adam, dense/kernel/Adam_1, dense/bias/Adam, dense/bias/Adam_1, dense_1/kernel/Adam, dense_1/kernel/Adam_1, dense_1/bias/Adam, dense_1/bias/Adam_1, dense_2/kernel/Adam, dense_2/kernel/Adam_1, dense_2/bias/Adam, dense_2/bias/Adam_1, dense_3/kernel/Adam, dense_3/kernel/Adam_1, dense_3/bias/Adam, dense_3/bias/Adam_1, dense_4/kernel/Adam, dense_4/kernel/Adam_1, dense_4/bias/Adam, dense_4/bias/Adam_1, dense_5/kernel/Adam, dense_5/kernel/Adam_1, dense_5/bias/Adam, dense_5/bias/Adam_1, ready: None\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23.455901: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23.455936: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23,605 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23,613 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23.680554: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:23.680591: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:24,177 - tensorflow - INFO - Saving checkpoints for 0 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:25,764 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[33mInstructions for updating:\u001b[0m\n",
      "\u001b[33mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:25,784 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:26.113238: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:26.113277: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2019-07-02 18:57:10 Training - Training image download completed. Training in progress.\u001b[33m2019-07-02 18:57:26,315 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:26,316 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:26,570 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:26,638 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:26,644 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:26,814 - tensorflow - INFO - loss = 2.3823955, step = 15\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:26,470 - tensorflow - INFO - loss = 20.751148, step = 0\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:30,032 - tensorflow - INFO - loss = 1.9236612, step = 193 (3.563 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:30,317 - tensorflow - INFO - loss = 2.0668314, step = 213 (3.503 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:33,845 - tensorflow - INFO - loss = 1.993006, step = 397 (3.813 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:34,103 - tensorflow - INFO - loss = 1.9190898, step = 411 (3.786 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:35,113 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[34mInstructions for updating:\u001b[0m\n",
      "\u001b[34mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:35,135 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:35,691 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:35,693 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:35,855 - tensorflow - INFO - Saving checkpoints for 501 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:35,971 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:36,052 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:36,061 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:36,274 - tensorflow - INFO - loss = 2.123454, step = 517\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:37.625703: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:37.625742: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,092 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,263 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,293 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:57:38\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:38,139 - tensorflow - WARNING - From model.py:84: map_and_batch (from tensorflow.contrib.data.python.ops.batching) is deprecated and will be removed in a future version.\u001b[0m\n",
      "\u001b[35mInstructions for updating:\u001b[0m\n",
      "\u001b[35mUse `tf.data.experimental.map_and_batch(...)`.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:38,160 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:37,940 - tensorflow - INFO - loss = 1.5050554, step = 603 (3.836 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,504 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,596 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-501\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,790 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,797 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,862 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,875 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,889 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,904 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,919 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,933 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,947 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,961 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,975 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:38,987 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39,007 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:57:39\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39,008 - tensorflow - INFO - Saving dict for global step 504: accuracy = 0.3828125, global_step = 504, loss = 1.7142254\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.018493: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.018531: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.034385: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.034418: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.047086: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39.047116: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:39,387 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 504: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-501\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:38,780 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:38,782 - tensorflow - INFO - Create CheckpointSaverHook.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:39,044 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:39,133 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:39,142 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:39,330 - tensorflow - INFO - loss = 1.5807397, step = 683\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:39,493 - tensorflow - INFO - loss = 1.7053361, step = 698 (3.220 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:41,076 - tensorflow - INFO - loss = 1.2815323, step = 859 (7.231 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:41,660 - tensorflow - INFO - loss = 1.3363622, step = 929 (3.720 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:42,264 - tensorflow - INFO - Saving checkpoints for 1007 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:42,928 - tensorflow - INFO - loss = 1.4979577, step = 1067 (3.598 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:45,862 - tensorflow - INFO - loss = 0.8671368, step = 1327 (4.202 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:47,156 - tensorflow - INFO - loss = 0.69803345, step = 1463 (4.228 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:43,452 - tensorflow - INFO - loss = 1.2085264, step = 1099 (3.959 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:44.177190: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:44.177247: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:44,386 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:46,164 - tensorflow - INFO - loss = 0.8586577, step = 1372 (5.088 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:47,252 - tensorflow - INFO - loss = 0.70665765, step = 1470 (3.800 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:47,681 - tensorflow - INFO - Saving checkpoints for 1511 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:49,890 - tensorflow - INFO - loss = 0.80551565, step = 1672 (4.028 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:49.562737: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:49.562806: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,019 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,193 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,219 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:57:50\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,333 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,402 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-1511\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,667 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,673 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,732 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,747 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,763 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,779 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,795 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,810 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,832 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,851 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:50,872 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:51,100 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:51,125 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:57:51\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:51,126 - tensorflow - INFO - Saving dict for global step 1513: accuracy = 0.821875, global_step = 1513, loss = 0.55641764\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:51,187 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 1513: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-1511\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:50,900 - tensorflow - INFO - loss = 0.46949467, step = 1752 (3.744 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:51,316 - tensorflow - INFO - loss = 0.2833712, step = 1791 (4.064 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:53,560 - tensorflow - INFO - Saving checkpoints for 2016 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:55,259 - tensorflow - INFO - loss = 0.34809524, step = 2184 (3.943 sec)\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2019-07-02 18:57:53,286 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:53,294 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:53,505 - tensorflow - INFO - loss = 0.53786814, step = 1997\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:54,418 - tensorflow - INFO - global_step/sec: 116.426\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:54,053 - tensorflow - INFO - loss = 0.27410915, step = 2067 (4.163 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:55,557 - tensorflow - INFO - global_step/sec: 92.1522\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:55.610185: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:55.610227: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:55,833 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:55,862 - tensorflow - INFO - loss = 0.23696926, step = 2252 (9.698 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:55,496 - tensorflow - INFO - loss = 0.17101145, step = 2183 (4.596 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:56,524 - tensorflow - INFO - global_step/sec: 111.719\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:57,386 - tensorflow - INFO - global_step/sec: 118.33\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:57,846 - tensorflow - INFO - loss = 0.297432, step = 2483 (4.341 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:57:58,058 - tensorflow - INFO - Saving checkpoints for 2519 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:58,161 - tensorflow - INFO - global_step/sec: 134.14\u001b[0m\n",
      "\u001b[33m2019-07-02 18:57:57,958 - tensorflow - INFO - loss = 0.22261004, step = 2500 (3.905 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:57:58,914 - tensorflow - INFO - loss = 0.33680683, step = 2607 (3.418 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:57:59,209 - tensorflow - INFO - loss = 0.19722089, step = 2633 (3.950 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:57:59,393 - tensorflow - INFO - global_step/sec: 98.2343\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:00,282 - tensorflow - INFO - global_step/sec: 117.024\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:00.511114: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:00.511152: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,101 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,280 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,310 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:58:01\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,436 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:01,474 - tensorflow - INFO - global_step/sec: 93.1346\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,537 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-2519\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,822 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,829 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,883 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,897 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,912 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,927 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,940 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,955 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,971 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:01,987 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,001 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,017 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,035 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:58:02\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,036 - tensorflow - INFO - Saving dict for global step 2523: accuracy = 0.9328125, global_step = 2523, loss = 0.2572934\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,095 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 2523: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-2519\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:02,755 - tensorflow - INFO - loss = 0.10886034, step = 3006 (3.546 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:02,166 - tensorflow - INFO - loss = 0.1980999, step = 2922 (4.321 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:02,525 - tensorflow - INFO - global_step/sec: 101.762\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:02,169 - tensorflow - INFO - loss = 0.11951938, step = 2923 (4.211 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:02,872 - tensorflow - INFO - Saving checkpoints for 3022 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:03,190 - tensorflow - INFO - loss = 0.16203174, step = 3047 (4.277 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:03,732 - tensorflow - INFO - global_step/sec: 96.953\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:04,650 - tensorflow - INFO - global_step/sec: 112.223\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:05.294540: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:05.294586: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:05,897 - tensorflow - INFO - loss = 0.17105265, step = 3327 (3.729 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:05,949 - tensorflow - INFO - global_step/sec: 110.019\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:06,681 - tensorflow - INFO - loss = 0.46112534, step = 3430 (4.514 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:06,758 - tensorflow - INFO - global_step/sec: 126.195\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:07,425 - tensorflow - INFO - loss = 0.23706487, step = 3512 (4.235 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:07,678 - tensorflow - INFO - global_step/sec: 110.885\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:05,531 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:05,774 - tensorflow - INFO - loss = 0.26396453, step = 3306 (9.912 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:07,005 - tensorflow - INFO - loss = 0.18114945, step = 3470 (4.250 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:07,534 - tensorflow - INFO - Saving checkpoints for 3525 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:08,963 - tensorflow - INFO - global_step/sec: 101.158\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:10.052482: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:10.052555: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:10,247 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:10,702 - tensorflow - INFO - loss = 0.38241562, step = 3856 (3.697 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:10,042 - tensorflow - INFO - loss = 0.28219756, step = 3776 (4.144 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:10,106 - tensorflow - INFO - global_step/sec: 100.582\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:10,660 - tensorflow - INFO - loss = 0.3720271, step = 3852 (3.979 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:10,909 - tensorflow - INFO - global_step/sec: 132.053\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:11,463 - tensorflow - INFO - loss = 0.5701285, step = 3957 (4.038 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:11,852 - tensorflow - INFO - global_step/sec: 111.309\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:11,947 - tensorflow - INFO - loss = 0.290744, step = 4010 (6.173 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:12,047 - tensorflow - INFO - Saving checkpoints for 4028 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:13,050 - tensorflow - INFO - global_step/sec: 101.825\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:14,148 - tensorflow - INFO - loss = 0.36808476, step = 4243 (4.106 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:14.674144: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:14.674195: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,065 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,234 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,259 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:58:15\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,373 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,460 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-4028\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:14,714 - tensorflow - INFO - loss = 0.12524854, step = 4294 (3.251 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,713 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,719 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,778 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,795 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,818 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,835 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,855 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,881 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,902 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:15,924 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:16,207 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:16,221 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:16,239 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:58:16\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:16,239 - tensorflow - INFO - Saving dict for global step 4033: accuracy = 0.9578125, global_step = 4033, loss = 0.13578819\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:16,392 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 4033: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-4028\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m2019-07-02 18:58:14,811 - tensorflow - INFO - loss = 0.10091041, step = 4308 (4.109 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:13,957 - tensorflow - INFO - global_step/sec: 114.702\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:14,963 - tensorflow - INFO - global_step/sec: 102.364\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:15,475 - tensorflow - INFO - loss = 0.10776798, step = 4357 (4.815 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:16,165 - tensorflow - INFO - global_step/sec: 88.1931\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:17,105 - tensorflow - INFO - Saving checkpoints for 4537 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:17,124 - tensorflow - INFO - global_step/sec: 111.582\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:18,429 - tensorflow - INFO - loss = 0.12154327, step = 4667 (3.617 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:18,543 - tensorflow - INFO - loss = 0.2301814, step = 4681 (4.395 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:18,279 - tensorflow - INFO - global_step/sec: 100.411\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:19.409090: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:19.409150: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:19,121 - tensorflow - INFO - loss = 0.23732409, step = 4754 (4.407 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:19,300 - tensorflow - INFO - global_step/sec: 104.838\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:19,896 - tensorflow - INFO - loss = 0.24655478, step = 4844 (4.421 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:19,750 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:20,032 - tensorflow - INFO - global_step/sec: 142.078\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:21,000 - tensorflow - INFO - global_step/sec: 113.655\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:21,904 - tensorflow - INFO - Saving checkpoints for 5059 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:22,560 - tensorflow - INFO - loss = 0.30728716, step = 5155 (4.132 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:22,307 - tensorflow - INFO - loss = 0.18942839, step = 5121 (3.764 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:22,076 - tensorflow - INFO - global_step/sec: 98.486\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:22,794 - tensorflow - INFO - global_step/sec: 146.188\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:23,898 - tensorflow - INFO - global_step/sec: 93.3371\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:24.436947: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:24.436984: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:23,520 - tensorflow - INFO - loss = 0.08530553, step = 5255 (4.399 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:24,675 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:24,686 - tensorflow - INFO - loss = 0.23593155, step = 5372 (12.739 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:26,250 - tensorflow - INFO - Saving checkpoints for 5560 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:26,579 - tensorflow - INFO - loss = 0.29647848, step = 5588 (4.018 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:27,556 - tensorflow - INFO - loss = 0.26057068, step = 5680 (4.037 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:24,019 - tensorflow - INFO - loss = 0.083893865, step = 5301 (4.124 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:24,870 - tensorflow - INFO - global_step/sec: 104.918\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:25,807 - tensorflow - INFO - global_step/sec: 113.168\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:26,630 - tensorflow - INFO - loss = 0.12549238, step = 5597 (4.323 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:26,656 - tensorflow - INFO - global_step/sec: 123.624\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:27,885 - tensorflow - INFO - global_step/sec: 96.0171\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:28,548 - tensorflow - INFO - loss = 0.100995004, step = 5767 (4.528 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:28.826452: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:28.826490: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,302 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,475 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:29,100 - tensorflow - INFO - global_step/sec: 93.7841\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,504 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:58:29\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,625 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,733 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-5560\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,979 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:29,986 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,042 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,057 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,073 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,088 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,104 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,120 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,136 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,151 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,166 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,191 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,214 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:58:30\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,215 - tensorflow - INFO - Saving dict for global step 5562: accuracy = 0.978125, global_step = 5562, loss = 0.07866716\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:30,335 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 5562: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-5560\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:30,824 - tensorflow - INFO - loss = 0.061454725, step = 6001 (4.246 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:30,211 - tensorflow - INFO - global_step/sec: 95.4734\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:31,268 - tensorflow - INFO - Saving checkpoints for 6062 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:31,102 - tensorflow - INFO - loss = 0.034523383, step = 6033 (4.471 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:31,213 - tensorflow - INFO - global_step/sec: 109.796\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:31,712 - tensorflow - INFO - loss = 0.070887014, step = 6109 (4.156 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:32,213 - tensorflow - INFO - global_step/sec: 104.922\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:32,682 - tensorflow - INFO - loss = 0.18118119, step = 6205 (4.134 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:33,273 - tensorflow - INFO - global_step/sec: 94.3663\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:33.684571: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:33.684613: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:33,906 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:34,404 - tensorflow - INFO - loss = 0.24486794, step = 6395 (9.718 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:34,434 - tensorflow - INFO - loss = 0.04760438, step = 6402 (3.609 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:35,001 - tensorflow - INFO - loss = 0.26872557, step = 6465 (3.899 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:36,073 - tensorflow - INFO - Saving checkpoints for 6595 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:35,619 - tensorflow - INFO - loss = 0.055709325, step = 6536 (3.906 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:38.275582: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:38.275622: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m2019-07-02 18:58:34,176 - tensorflow - INFO - global_step/sec: 117.36\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:34,972 - tensorflow - INFO - global_step/sec: 126.909\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:35,816 - tensorflow - INFO - global_step/sec: 123.242\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:36,821 - tensorflow - INFO - global_step/sec: 106.421\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:37,052 - tensorflow - INFO - loss = 0.047711678, step = 6698 (4.370 sec)\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:38,520 - tensorflow - INFO - loss = 0.1076369, step = 6833 (4.086 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:38,188 - tensorflow - INFO - global_step/sec: 83.4335\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:38,918 - tensorflow - INFO - global_step/sec: 143.811\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:38,590 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:38,968 - tensorflow - INFO - loss = 0.23757617, step = 6899 (3.968 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:39,717 - tensorflow - INFO - global_step/sec: 132.641\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:40,333 - tensorflow - INFO - loss = 0.12026495, step = 7064 (5.929 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:39,676 - tensorflow - INFO - loss = 0.14566714, step = 6992 (4.057 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:40,691 - tensorflow - INFO - global_step/sec: 105.702\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:41,039 - tensorflow - INFO - Saving checkpoints for 7150 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:41,580 - tensorflow - INFO - loss = 0.1167277, step = 7196 (4.528 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:41,660 - tensorflow - INFO - global_step/sec: 104.297\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:42,313 - tensorflow - INFO - loss = 0.6026316, step = 7275 (3.793 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:42,961 - tensorflow - INFO - loss = 0.083959654, step = 7336 (3.993 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:42,736 - tensorflow - INFO - global_step/sec: 94.7848\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:43.238686: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:43.238730: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:43,520 - tensorflow - INFO - global_step/sec: 128.856\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:43,729 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:43,908 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:43,938 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:58:43\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,059 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,131 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-7150\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,395 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,402 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,456 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,472 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,487 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:43,867 - tensorflow - INFO - loss = 0.14749064, step = 7431 (4.191 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:44,819 - tensorflow - INFO - global_step/sec: 91.6271\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,502 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,517 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,531 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,546 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,560 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,575 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,591 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,612 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:58:44\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,613 - tensorflow - INFO - Saving dict for global step 7155: accuracy = 0.975, global_step = 7155, loss = 0.10028534\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:44,701 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 7155: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-7150\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:45,603 - tensorflow - INFO - global_step/sec: 137.672\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:45,659 - tensorflow - INFO - loss = 0.19847085, step = 7638 (4.078 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:45,723 - tensorflow - INFO - Saving checkpoints for 7656 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:46,615 - tensorflow - INFO - global_step/sec: 106.769\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:47,913 - tensorflow - INFO - global_step/sec: 81.6276\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:48.025217: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:48.025299: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:48,333 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:48,651 - tensorflow - INFO - global_step/sec: 144.95\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:46,198 - tensorflow - INFO - loss = 0.17042437, step = 7702 (3.885 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:47,193 - tensorflow - INFO - loss = 0.41967982, step = 7793 (4.232 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:47,978 - tensorflow - INFO - loss = 0.22479671, step = 7855 (4.111 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:49,523 - tensorflow - INFO - global_step/sec: 115.814\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:49,927 - tensorflow - INFO - loss = 0.05930982, step = 8105 (4.268 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:50,116 - tensorflow - INFO - loss = 0.15429987, step = 8119 (9.783 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:50,365 - tensorflow - INFO - Saving checkpoints for 8161 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:50,240 - tensorflow - INFO - loss = 0.20079535, step = 8137 (4.042 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:50,911 - tensorflow - INFO - loss = 0.07909021, step = 8226 (3.718 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:50,489 - tensorflow - INFO - global_step/sec: 127.312\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:51,709 - tensorflow - INFO - global_step/sec: 95.0997\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:51,733 - tensorflow - INFO - loss = 0.25239187, step = 8296 (3.756 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:52,776 - tensorflow - INFO - global_step/sec: 96.5953\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:52.691138: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:52.691177: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:52,952 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:53,588 - tensorflow - INFO - global_step/sec: 128.012\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:54,304 - tensorflow - INFO - loss = 0.06410249, step = 8582 (4.064 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:54,945 - tensorflow - INFO - loss = 0.13471094, step = 8673 (4.034 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:54,315 - tensorflow - INFO - loss = 0.1177061, step = 8583 (4.389 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:54,432 - tensorflow - INFO - global_step/sec: 124.475\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:54,877 - tensorflow - INFO - Saving checkpoints for 8667 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:55,286 - tensorflow - INFO - global_step/sec: 122.94\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:56,335 - tensorflow - INFO - global_step/sec: 98.1418\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:57.100388: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:57.100428: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:57,637 - tensorflow - INFO - global_step/sec: 97.5296\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:57,809 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:57,986 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,016 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:58:58\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,140 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,241 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-8667\u001b[0m\n",
      "\u001b[34m2019-07-02 18:58:57,948 - tensorflow - INFO - loss = 0.086749665, step = 8975 (3.643 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:58,608 - tensorflow - INFO - global_step/sec: 103.056\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:58,674 - tensorflow - INFO - loss = 0.06765793, step = 9045 (4.359 sec)\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,739 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,745 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,806 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,828 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,845 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,860 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,877 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,903 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,927 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:58,946 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,218 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,235 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,253 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:58:59\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,253 - tensorflow - INFO - Saving dict for global step 8670: accuracy = 0.9828125, global_step = 8670, loss = 0.050361134\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,366 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 8670: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-8667\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[35m2019-07-02 18:58:55,783 - tensorflow - INFO - loss = 0.039712176, step = 8762 (4.049 sec)\u001b[0m\n",
      "\u001b[35m2019-07-02 18:58:59,427 - tensorflow - INFO - loss = 0.23454589, step = 9121 (3.644 sec)\u001b[0m\n",
      "\u001b[33m2019-07-02 18:58:59,526 - tensorflow - INFO - loss = 0.104218215, step = 9138 (4.581 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:58:59,552 - tensorflow - INFO - global_step/sec: 110.101\u001b[0m\n",
      "\u001b[31m2019-07-02 18:58:59,723 - tensorflow - INFO - Saving checkpoints for 9169 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[32m2019-07-02 18:59:00,622 - tensorflow - INFO - global_step/sec: 101.887\u001b[0m\n",
      "\u001b[34m2019-07-02 18:59:01,727 - tensorflow - INFO - loss = 0.05827143, step = 9360 (3.779 sec)\u001b[0m\n",
      "\u001b[32m2019-07-02 18:59:01,678 - tensorflow - INFO - global_step/sec: 96.6374\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:02.260222: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:02.260274: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:59:01,979 - tensorflow - INFO - Loss for final step: 0.08380407.\u001b[0m\n",
      "\u001b[34m2019-07-02 18:59:02,159 - tensorflow - INFO - Loss for final step: 0.20779562.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:59:02,057 - tensorflow - INFO - Loss for final step: 0.28052288.\u001b[0m\n",
      "\u001b[33m2019-07-02 18:59:02,519 sagemaker_tensorflow_container.training INFO     master algo-1 is still up, waiting for it to exit\u001b[0m\n",
      "\u001b[32m2019-07-02 18:59:02,395 - tensorflow - INFO - Loss for final step: 0.09744049.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:02,521 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:02,534 - tensorflow - INFO - Saving checkpoints for 9382 into s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt.\u001b[0m\n",
      "\u001b[35m2019-07-02 18:59:03,228 sagemaker_tensorflow_container.training INFO     master algo-1 is still up, waiting for it to exit\u001b[0m\n",
      "\u001b[34m2019-07-02 18:59:03,049 sagemaker_tensorflow_container.training INFO     master algo-1 is still up, waiting for it to exit\u001b[0m\n",
      "\u001b[32m2019-07-02 18:59:03,729 sagemaker_tensorflow_container.training INFO     master algo-1 is still up, waiting for it to exit\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:04.808290: E tensorflow/core/platform/s3/aws_logging.cc:60] No response body. Response code: 404\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:04.808329: W tensorflow/core/platform/s3/aws_logging.cc:57] If the signature check failed. This could be because of a time skew. Attempting to adjust the signer.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,041 - tensorflow - INFO - Skip the current checkpoint eval due to throttle secs (10 secs).\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,196 - tensorflow - INFO - Calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,356 - tensorflow - INFO - Done calling model_fn.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,383 - tensorflow - INFO - Starting evaluation at 2019-07-02-18:59:05\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,582 - tensorflow - INFO - Graph was finalized.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,670 - tensorflow - INFO - Restoring parameters from s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-9382\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,861 - tensorflow - INFO - Running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,867 - tensorflow - INFO - Done running local_init_op.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,920 - tensorflow - INFO - Evaluation [1/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,934 - tensorflow - INFO - Evaluation [2/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,947 - tensorflow - INFO - Evaluation [3/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,960 - tensorflow - INFO - Evaluation [4/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,974 - tensorflow - INFO - Evaluation [5/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:05,987 - tensorflow - INFO - Evaluation [6/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,000 - tensorflow - INFO - Evaluation [7/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,013 - tensorflow - INFO - Evaluation [8/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,035 - tensorflow - INFO - Evaluation [9/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,051 - tensorflow - INFO - Evaluation [10/10]\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,067 - tensorflow - INFO - Finished evaluation at 2019-07-02-18:59:06\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,067 - tensorflow - INFO - Saving dict for global step 9382: accuracy = 0.98125, global_step = 9382, loss = 0.06302488\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,311 - tensorflow - INFO - Saving 'checkpoint_path' summary for global step 9382: s3://com.climate.production.analytics/dsw/scratch/sagemaker/models/peta_sage_hack/20190702-184348/output/petastorm-sagemaker-2019-07-02-18-54-05-832/model/model.ckpt-9382\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,387 - tensorflow - DEBUG - Calling exporter with the `is_the_final_export=True`.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:06,398 - tensorflow - INFO - Loss for final step: 0.13780728.\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:07,058 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[31mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[31mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[31m2019-07-02 18:59:07,059 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\u001b[32m2019-07-02 19:00:20,556 sagemaker_tensorflow_container.training INFO     master algo-1 is down, stopping parameter server\u001b[0m\n",
      "\u001b[32m2019-07-02 19:00:20,556 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[32mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[32mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[32m2019-07-02 19:00:20,556 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\u001b[35m2019-07-02 19:00:20,866 sagemaker_tensorflow_container.training INFO     master algo-1 is down, stopping parameter server\u001b[0m\n",
      "\u001b[35m2019-07-02 19:00:20,867 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[35mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[35mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[35m2019-07-02 19:00:20,867 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\u001b[34m2019-07-02 19:00:21,251 sagemaker_tensorflow_container.training INFO     master algo-1 is down, stopping parameter server\u001b[0m\n",
      "\u001b[34m2019-07-02 19:00:21,252 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[34mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[34mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[34m2019-07-02 19:00:21,252 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\u001b[33m2019-07-02 19:01:23,747 sagemaker_tensorflow_container.training INFO     master algo-1 is down, stopping parameter server\u001b[0m\n",
      "\u001b[33m2019-07-02 19:01:23,747 sagemaker_tensorflow_container.training WARNING  No model artifact is saved under path /opt/ml/model. Your training job will not save any model files to S3.\u001b[0m\n",
      "\u001b[33mFor details of how to construct your training script see:\u001b[0m\n",
      "\u001b[33mhttps://github.com/aws/sagemaker-python-sdk/tree/master/src/sagemaker/tensorflow#adapting-your-local-tensorflow-script\u001b[0m\n",
      "\u001b[33m2019-07-02 19:01:23,747 sagemaker-containers INFO     Reporting training SUCCESS\u001b[0m\n",
      "\n",
      "2019-07-02 19:02:18 Uploading - Uploading generated training model\n",
      "2019-07-02 19:02:18 Completed - Training job completed\n",
      "Billable seconds: 1850\n",
      "CPU times: user 1.59 s, sys: 121 ms, total: 1.71 s\n",
      "Wall time: 8min 24s\n"
     ]
    }
   ],
   "source": [
    "kwargs['train_instance_count'] = 5\n",
    "mnist_estimator = TensorFlow(**kwargs)\n",
    "%time  mnist_estimator.fit(inputs=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Final accuracy is 0.95ish as well - this time wall time = 8min and 24s."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}