087_initialization_experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "def ReLU(x):\n",
    "    return np.maximum(0, x)\n",
    "\n",
    "def tanh(x):\n",
    "    return np.tanh(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZU0lEQVR4nO3df7DddX3n8efLBJAtWlBSliaMoZpqo7tGjYEdW5eiQsDdBWesA1sl61BjK3R1p7tjdHYWirKrM7V0mSK7UVKCv4BRK1mIZTMU17EjP4IgEBC55ccmKT+uhp9FoND3/nE+V8/mnuSe3Htzz7n3Ph8z38n3fL6f7/d8vu977nmd749zk6pCkqSXDHoAkqThYCBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQfi7JA0neOehxDBvrMp41GS9JJXnNoMcxTGZjTeZ0ICQ5O8nWJM8luXTQ4xkGSQ5KckmSB5M8leS2JCcNelyDluTLSR5K8mSSHyf5vUGPaVgkWZbk2SRfHvRYBi3Jd1otnm7TPYMe03Sa04EA/B3waWDDoAfSS5KFA3jahcB24F8Cvwz8Z+DKJEsHMJaeBlSX/wYsraqXA/8G+HSStwxgHD0NqCZjLgJuHuDz95RkwYCe+uyqOqRNrx3QGHqaak3mdCBU1Ter6lvAT/dlvSSrknw/yePtU+OfJzmwLbsoyed2678pyX9o87+a5BtJRpPcn+Tfd/U7N8nX26fRJ4F/N+Wd3EdV9fdVdW5VPVBV/1hVVwP3AxO++c3xumyrqufGHrbp1ROtN5dr0sZxGvA4cN0+rPPuJLe2o63tSc7tWnZNkj/crf/tSd7T5l+XZEuSXUnuSfK+rn6XJrk4yeYkfw/89lT3b6bMmppU1Zyf6BwlXDpBnweAd7b5twDH0vk0vRS4G/hYW7aKzpHHS9rjw4FngCPoBOwtwH8BDgR+DbgPOLH1PRf4B+DU1vfgIajNEcCzwOvme12Az7cxF/AD4JD5XBPg5cCPgSVtPF/eS98CXtPmjwP+WRv3PwceAU5ty94H3Ni13hvpfGA7EPglOkevH2z1fBPwE2B563sp8ATwtrbtlw6gJt8BRtu4/gY4bi7VZE4fIUxWVd1SVTdU1QtV9QDwP+mcYqGqbqLzA3hH634a8J2qegR4K7Coqs6rquer6j7gC63PmO9X1beq8+n8ZzO1T70kOQD4CrCxqn40Uf+5Xpeq+gjwMuC3gG8Cz+19jTlfk08Bl1TVjn1Zqaq+U1V3tHHfDnyNVhNgE/DrSZa1xx8Arqiq54F/BTxQVX/R6nkr8A3gd7o2f1VV/U3b9rNT2blJ+jid8F4MrAf+V5IJjyRnS03mZSAk+XbXRaHf7bH815NcneThdrj+X+l8uhuzEXh/m38/8KU2/yrgV9vpg8eTPA58ks4nwjHbp32HJiHJS+iM+3ng7NY27+tSVS9W1ffofCr+g/lakyQrgHcCF/RYtq2rJr/VY/kxSa5vp8KeAH6fVpP2hnUF8P72Gjyd/78mx+xWk98F/mnX5gf6OqmqG6vqqap6rqo20jlKOHmu1GSQF6oGpqomuqvmYuBW4PSqeirJx4D3di3/MnBnkjcCvwF8q7VvB+6vqmXs2cD/vGySAJfQefM5uar+AazLbhYCr57HNTmOzimw/9t5uXAIsCDJ8qp6/QTrfhX4c+Ckqno2yZ8xPiS/BHwPeKaqvt/atwP/p6retZdtD9vrpIDMlZrM6SOEJAuTvBRYQOfF/NL0d7fGy4AngaeTvA74g+6F7RD6Zjo/wG90Hc7fBDyV5ONJDk6yIMkbkrx12nZqelxM583pX+/jqYg5WZckv5LktCSHtLGdSOdTWj8XUudkTeicDnk1sKJN/wO4Bjixj3VfBuxqb3yrgH/bvbC92f0j8Dl+8UkY4Go6p04+kOSANr01yW9MfXemLsmhSU4cex9pR4xvB/6qj9VnRU3mdCDQuaXyZ8A6OofrP2ttE/mPdH5gT9E5r3tFjz4b6Vwk+vkPr6pepHPObwWdO3d+AnyRzu2dQyHJq4AP0xnjw3s7HdLDXK1L0Xkj3wE8BvwJnQvDm/pYd07WpKqeqaqHxybgaeDZqhrtY/WPAOcleYrORfMre/S5jE5Nfv7dhqp6CjiBznWUvwMeBj4LHDSlnZk+B9C5QWXsovIf0rkw/OM+1p0VNUnVsB2BzQ5J3k7nB/eqsog/Z13GsybjJTkDWFtVvznosQyLYajJXD9C2C/SuTvno8AX/QX/BesynjUZL8k/ofOJef2gxzIshqUmBsI+aufuHgeOBP5swMMZGtZlPGsyXrs+M0rnPvyvDng4Q2GYauIpI0kS4BGCJKmZtd9DOPzww2vp0qWDHsZ+dcstt/ykqhb1238+1AT2rS7WpLf5UBdr0tve6jJrA2Hp0qVs3bp10MPYr5I8uC/950NNYN/qYk16mw91sSa97a0unjKSJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAXMgEJauu4al664Z9DCmxVzal+kynTWxvuNZk7ltX3++sz4QJEnTw0CQJAEGgiSpMRAkSYCBIElqDARJEtBHICR5aZKbkvwwybYkf9zaL01yf5Lb2rSitSfJhUlGktye5M1d21qT5N42relqf0uSO9o6FybJ/tjZ+cRbCSXtq37+x7TngOOr6ukkBwDfS/Lttuw/VdXXd+t/ErCsTccAFwPHJHkFcA6wEijgliSbquqx1udDwI3AZmA18G0kSTNmwiOE6ni6PTygTbWXVU4BLmvr3QAcmuRI4ERgS1XtaiGwBVjdlr28qm6oqgIuA06dwj5Jkiahr2sISRYkuQ14lM6b+o1t0fnttNAFSQ5qbYuB7V2r72hte2vf0aO91zjWJtmaZOvo6Gg/Q5ck9amvQKiqF6tqBbAEWJXkDcAngNcBbwVeAXx8v43yF+NYX1Urq2rlokWL9vfTSdK8sk93GVXV48D1wOqqeqidFnoO+AtgVeu2Eziqa7UlrW1v7Ut6tEuSZlA/dxktSnJomz8YeBfwo3bun3ZH0KnAnW2VTcAZ7W6jY4Enquoh4FrghCSHJTkMOAG4ti17MsmxbVtnAFdN725KkibSz11GRwIbkyygEyBXVtXVSf46ySIgwG3A77f+m4GTgRHgGeCDAFW1K8mngJtbv/Oqaleb/whwKXAwnbuLvMNIkmbYhIFQVbcDb+rRfvwe+hdw1h6WbQA29GjfCrxhorFIkvYfv6ksSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQpFnJ/+9C+4OBIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSV2Wrrtm3n7xz0CQNCfM5zfy6WIgSJKAPgIhyUuT3JTkh0m2Jfnj1n50khuTjCS5IsmBrf2g9nikLV/ata1PtPZ7kpzY1b66tY0kWTf9uylJmkg/RwjPAcdX1RuBFcDqJMcCnwUuqKrXAI8BZ7b+ZwKPtfYLWj+SLAdOA14PrAY+n2RBkgXARcBJwHLg9NZXkjSDJgyE6ni6PTygTQUcD3y9tW8ETm3zp7THtOXvSJLWfnlVPVdV9wMjwKo2jVTVfVX1PHB56ytJmkF9XUNon+RvAx4FtgB/CzxeVS+0LjuAxW1+MbAdoC1/Anhld/tu6+ypvdc41ibZmmTr6OhoP0OXJPWpr0CoqheragWwhM4n+tft11HteRzrq2plVa1ctGjRIIYgSXPWPt1lVFWPA9cD/wI4NMnCtmgJsLPN7wSOAmjLfxn4aXf7buvsqV2SNIP6uctoUZJD2/zBwLuAu+kEw3tbtzXAVW1+U3tMW/7XVVWt/bR2F9LRwDLgJuBmYFm7a+lAOheeN03HzkmS+rdw4i4cCWxsdwO9BLiyqq5OchdweZJPA7cCl7T+lwBfSjIC7KLzBk9VbUtyJXAX8AJwVlW9CJDkbOBaYAGwoaq2TdseSpL6MmEgVNXtwJt6tN9H53rC7u3PAr+zh22dD5zfo30zsLmP8UqS9hO/qSxJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkoA+AiHJUUmuT3JXkm1JPtraz02yM8ltbTq5a51PJBlJck+SE7vaV7e2kSTrutqPTnJja78iyYHTvaOSpL3r5wjhBeCPqmo5cCxwVpLlbdkFVbWiTZsB2rLTgNcDq4HPJ1mQZAFwEXASsBw4vWs7n23beg3wGHDmNO2fJKlPEwZCVT1UVT9o808BdwOL97LKKcDlVfVcVd0PjACr2jRSVfdV1fPA5cApSQIcD3y9rb8ROHWyOyRJmpx9uoaQZCnwJuDG1nR2ktuTbEhyWGtbDGzvWm1Ha9tT+yuBx6vqhd3aez3/2iRbk2wdHR3dl6FLkibQdyAkOQT4BvCxqnoSuBh4NbACeAj43H4ZYZeqWl9VK6tq5aJFi/b300nSvLKwn05JDqATBl+pqm8CVNUjXcu/AFzdHu4EjupafUlrYw/tPwUOTbKwHSV095ckzZB+7jIKcAlwd1X9aVf7kV3d3gPc2eY3AaclOSjJ0cAy4CbgZmBZu6PoQDoXnjdVVQHXA+9t668BrprabkmS9lU/RwhvAz4A3JHkttb2STp3Ca0ACngA+DBAVW1LciVwF507lM6qqhcBkpwNXAssADZU1ba2vY8Dlyf5NHArnQCSJM2gCQOhqr4HpMeizXtZ53zg/B7tm3utV1X30bkLSZI0IH5TWZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAX0EQpKjklyf5K4k25J8tLW/IsmWJPe2fw9r7UlyYZKRJLcneXPXtta0/vcmWdPV/pYkd7R1LkzS6/9wliTtR/0cIbwA/FFVLQeOBc5KshxYB1xXVcuA69pjgJOAZW1aC1wMnQABzgGOAVYB54yFSOvzoa71Vk991yRJ+2LCQKiqh6rqB23+KeBuYDFwCrCxddsInNrmTwEuq44bgEOTHAmcCGypql1V9RiwBVjdlr28qm6oqgIu69qWJGmG7NM1hCRLgTcBNwJHVNVDbdHDwBFtfjGwvWu1Ha1tb+07erT3ev61SbYm2To6OrovQ5ckTaDvQEhyCPAN4GNV9WT3svbJvqZ5bONU1fqqWllVKxctWrS/n06S5pW+AiHJAXTC4CtV9c3W/Eg73UP799HWvhM4qmv1Ja1tb+1LerRLkmZQP3cZBbgEuLuq/rRr0SZg7E6hNcBVXe1ntLuNjgWeaKeWrgVOSHJYu5h8AnBtW/ZkkmPbc53RtS1J0gxZ2EeftwEfAO5Icltr+yTwGeDKJGcCDwLva8s2AycDI8AzwAcBqmpXkk8BN7d+51XVrjb/EeBS4GDg222SJM2gCQOhqr4H7Ol7Ae/o0b+As/awrQ3Ahh7tW4E3TDQWSdL+4zeVJUmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkS0EcgJNmQ5NEkd3a1nZtkZ5Lb2nRy17JPJBlJck+SE7vaV7e2kSTrutqPTnJja78iyYHTuYOSpP70c4RwKbC6R/sFVbWiTZsBkiwHTgNe39b5fJIFSRYAFwEnAcuB01tfgM+2bb0GeAw4cyo7JEmanAkDoaq+C+zqc3unAJdX1XNVdT8wAqxq00hV3VdVzwOXA6ckCXA88PW2/kbg1H3cB0nSNJjKNYSzk9zeTikd1toWA9u7+uxobXtqfyXweFW9sFt7T0nWJtmaZOvo6OgUhi5J2t1kA+Fi4NXACuAh4HPTNqK9qKr1VbWyqlYuWrRoJp5SkuaNhZNZqaoeGZtP8gXg6vZwJ3BUV9clrY09tP8UODTJwnaU0N1fkjSDJnWEkOTIrofvAcbuQNoEnJbkoCRHA8uAm4CbgWXtjqID6Vx43lRVBVwPvLetvwa4ajJjkiRNzYRHCEm+BhwHHJ5kB3AOcFySFUABDwAfBqiqbUmuBO4CXgDOqqoX23bOBq4FFgAbqmpbe4qPA5cn+TRwK3DJtO2dJKlvEwZCVZ3eo3mPb9pVdT5wfo/2zcDmHu330bkLSZI0QH5TWZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAX0EQpINSR5NcmdX2yuSbElyb/v3sNaeJBcmGUlye5I3d62zpvW/N8marva3JLmjrXNhkkz3TkqSJtbPEcKlwOrd2tYB11XVMuC69hjgJGBZm9YCF0MnQIBzgGOAVcA5YyHS+nyoa73dn0uSNAMmDISq+i6wa7fmU4CNbX4jcGpX+2XVcQNwaJIjgROBLVW1q6oeA7YAq9uyl1fVDVVVwGVd25IkzaDJXkM4oqoeavMPA0e0+cXA9q5+O1rb3tp39GiXJM2wKV9Ubp/saxrGMqEka5NsTbJ1dHR0Jp5SkuaNyQbCI+10D+3fR1v7TuCorn5LWtve2pf0aO+pqtZX1cqqWrlo0aJJDl2S1MtkA2ETMHan0Brgqq72M9rdRscCT7RTS9cCJyQ5rF1MPgG4ti17Msmx7e6iM7q2JUmaQQsn6pDka8BxwOFJdtC5W+gzwJVJzgQeBN7Xum8GTgZGgGeADwJU1a4knwJubv3Oq6qxC9UfoXMn08HAt9skSZphEwZCVZ2+h0Xv6NG3gLP2sJ0NwIYe7VuBN0w0DknS/uU3lSVJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJKaKQVCkgeS3JHktiRbW9srkmxJcm/797DWniQXJhlJcnuSN3dtZ03rf2+SNVPbJUnSZEzHEcJvV9WKqlrZHq8DrquqZcB17THAScCyNq0FLoZOgADnAMcAq4BzxkJEkjRz9scpo1OAjW1+I3BqV/tl1XEDcGiSI4ETgS1VtauqHgO2AKv3w7gkSXsx1UAo4H8nuSXJ2tZ2RFU91OYfBo5o84uB7V3r7mhte2ofJ8naJFuTbB0dHZ3i0CVJ3RZOcf3frKqdSX4F2JLkR90Lq6qS1BSfo3t764H1ACtXrpy27Wp+WbruGh74zLsHPYxJWbrumkEPQXPYlI4Qqmpn+/dR4C/pXAN4pJ0Kov37aOu+Eziqa/UlrW1P7ZKkGTTpQEjyS0leNjYPnADcCWwCxu4UWgNc1eY3AWe0u42OBZ5op5auBU5Icli7mHxCa5MkzaCpnDI6AvjLJGPb+WpV/VWSm4Erk5wJPAi8r/XfDJwMjADPAB8EqKpdST4F3Nz6nVdVu6YwLjVjpxdm6+kRaZBm8+/PZE8tTjoQquo+4I092n8KvKNHewFn7WFbG4ANkx2L5ibPl0szy28qS2LpumsMYBkIkqQOA0GSBBgIkqRmql9Mk2al2XYHief398zaTB+PECRJwBw6Qphtn/i6+QlH0jDwCGEe8JZCafJm2+/PVMZqIGjozLZfwLnEus9vc+aUkTQZw36q0TdozSQDQdKsZFhOPwNhHvHTsDR5w/7/aEzH78+cu4Yw284/z6axzmXD+LoZ1HiGsRaaGR4hzEPD9klnmN58uscy0zUapjoMu0GGJQzXUfZ01sJAGJBB//IPywt70HXYm5mq0bDWYFheI8NoGGqzP143czYQhu1T8LAa1At7WN8Ee9kfRw2zcf+H4fdp2Oo2135/5mwgwHC9kMcM2wt6TK9xTWfdhnW/99Vc2Y/JGMSHrNlS7/19qnGm6jCnA2HM7sX0Rd2f2Tpu7T97ek088Jl3T/kD2O7rz9bX32wdN8yTQNjd3n5gk31hz+YXgTRV3a//qf4u+Ls0OEMTCElWA/8dWAB8sao+M4hxTOcLW5Jmk6H4HkKSBcBFwEnAcuD0JMsHOypJml+GIhCAVcBIVd1XVc8DlwOnDHhMkjSvpKoGPQaSvBdYXVW/1x5/ADimqs7erd9aYG17+FrgHuBw4CczONyZMLZPr6qqRf2ulGQUeHC3bcwV3fvTd12sSW9ddZlrNYGp//7M5ZrAXuoyNNcQ+lFV64H13W1JtlbVygENab+Y7D51/5DnWl2syXhT2Z+xusy1msDUXyvzuSbDcspoJ3BU1+MlrU2SNEOGJRBuBpYlOTrJgcBpwKYBj0mS5pWhOGVUVS8kORu4ls5tpxuqalufq6+fuMusMx37NNfqYk3Gsya9TXWf5m1NhuKisiRp8IbllJEkacAMBEkSMMsDIcnqJPckGUmybtDjmaokG5I8muTOKWzDmozfhjXpvR3rMn4b87smVTUrJzoXn/8W+DXgQOCHwPJBj2uK+/R24M3AndbEmuyvmlgXa7KnaTYfIcy5P3dRVd8Fdk1hE9ZkPGvSm3UZb97XZDYHwmJge9fjHa1tPrMm41mT3qzLePO+JrM5ECRJ02g2B4J/7mI8azKeNenNuow372symwPBP3cxnjUZz5r0Zl3Gm/c1mbWBUFUvAGN/7uJu4Mrq/89dDKUkXwO+D7w2yY4kZ+7L+tZkPGvSm3UZz5r4pyskSc2sPUKQJE0vA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWr+H+Hkam2IMjbIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "input_data = np.random.randn(1000, 100)  # 1000개의 데이터\n",
    "node_num = 100  # 각 은닉층의 노드(뉴런) 수\n",
    "hidden_layer_size = 5  # 은닉층이 5개\n",
    "activations = {}  # 이곳에 활성화 결과를 저장\n",
    "\n",
    "x = input_data\n",
    "\n",
    "for i in range(hidden_layer_size):\n",
    "    if i != 0:\n",
    "        x = activations[i-1]\n",
    "\n",
    "    w = np.random.randn(node_num, node_num) * 1\n",
    "#     w = np.random.randn(node_num, node_num) * 0.01\n",
    "#     w = np.random.randn(node_num, node_num) * np.sqrt(1.0 / node_num)\n",
    "#     w = np.random.randn(node_num, node_num) * np.sqrt(2.0 / node_num)\n",
    "\n",
    "    a = np.dot(x, w)\n",
    "    z = sigmoid(a)\n",
    "    # z = ReLU(a)\n",
    "    # z = tanh(a)\n",
    "\n",
    "    activations[i] = z\n",
    "\n",
    "for i, a in activations.items():\n",
    "    plt.subplot(1, len(activations), i+1)\n",
    "    plt.title(str(i+1) + \"-layer\")\n",
    "    if i != 0:\n",
    "        plt.yticks([], [])\n",
    "    # plt.xlim(0.1, 1)\n",
    "    # plt.ylim(0, 7000)\n",
    "    plt.hist(a.flatten(), 30, range=(0, 1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEICAYAAABfz4NwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAV00lEQVR4nO3dfZBldX3n8ffHGRA2iGCYsGSGckicaNBdUUZgy8Q1ojBgEtgqpWBVZi3i7AbI6tZurWNqa3F92NWqZDFUkN1ZmTBoFCg1MgsYdgplU6YEZgiGxyAdHnZmwkPr8GQQCPjdP+6v8Tp9e/r2dE/f7nvfr6pbfc7v/M65v/Pt2/dzz0N3p6qQJOllgx6AJGlhMBAkSYCBIElqDARJEmAgSJIaA0GSBBgIL0nyYJJ3DnocC4k16c26TJakkrxm0ONYSBZjTYY6EJKcn2RbkueSXDbo8QxakpcnuTTJQ0meTvK9JKcMelwLQZIvJXk4yVNJvp/kdwY9poUiyaokzyb50qDHMmhJbmy1+FF73DvoMc2loQ4E4O+ATwEbBz2QXpIsneenXApsB/458ErgPwFXJVk5z+OY0gBqMuG/ASur6mDgt4FPJTl2QGOZZIB1AbgY2DrA5+8pyZIBPfX5VXVQe7x2QGPoabY1GepAqKqvV9U3gB/OZL0kxyX5bpIn2qfGP06yf1t2cZI/3K3/5iT/rk3/YpKvJRlP8kCSf9vV7+NJvto+jT4F/KtZ7+QMVNXfV9XHq+rBqvpJVV0DPABM+8Y3rDWZUFV3VdVzE7Pt8cvTrTfsdUlyJvAEcMMM1nl3ktva0db2JB/vWnZtkt/brf/tSf5Fm35dki1JdiW5N8kZXf0uS3JJkuuS/D3wG7Pdv/myaGpSVUP/oHOUcNk0fR4E3tmmjwVOoPOJeiVwD/CRtuw4OkceL2vzhwHPAIfTCdhbgf8M7A/8EnA/cHLr+3HgH4DTW98DB1yXw4FngddZkwL4fBt3AX8FHDTKdQEOBr4PrGjj+dIe+hbwmjb9duCftHH/U+BR4PS27Azg5q713kjnA9v+wM/ROYL9YKvnm4AfAEe3vpcBTwJvbds+YAA1uREYb+P6S+Dtw1SToT5C2FtVdWtV3VRVL1TVg8D/pHOahaq6hc434MTW/Uzgxqp6FHgLsKyqPlFVz1fV/cD/an0mfLeqvlGdT+g/nq992l2S/YA/BTZV1d9M138UalJV5wKvAH4d+Drw3J7XGPq6fBK4tKp2zGSlqrqxqu5o474d+AqtJsBm4FeSrGrzHwCurKrngd8EHqyqP2n1vA34GvDers1fXVV/2bb97Gx2bi99lE54Lwc2AP87ybRHkoulJiMZCEm+2XVR6H09lv9KkmuSPNIO1/8rnU93EzYB72/T7we+2KZfDfxiO33wRJIngN+n84lwwvY536EZSvIyOmN+Hji/tY10TSZU1YtV9R06n4p/d1TrkuQY4J3AhT2W3dVVk1/vsfz4JN9up8KeBP4NrSbtDetK4P3tdXgWP1uT43eryfuAf9y1+YG+Vqrq5qp6uqqeq6pNdI4STh2WmgzyQtXAVNV0d9ZcAtwGnFVVTyf5CPCeruVfAu5M8kbgV4FvtPbtwANVtYqpDfTPyyYJcCmdN55Tq+ofYLRrMoWlwC+PcF3eTucU2P/rvGQ4CFiS5Oiqev00634Z+GPglKp6NsnnmBySXwS+AzxTVd9t7duB/1tV79rDthfaa6WADEtNhvoIIcnSJAcAS+i8mA9If3drvAJ4CvhRktcBv9u9sB1Cb6XzDfxa1+H8LcDTST6a5MAkS5K8Iclb5mynZu8SOm9MvzXD0xBDW5Mkv5DkzCQHtfGdTOdTWj8XUoe1LhvoXFQ/pj3+B3AtcHIf674C2NXe+I4D/mX3wvZm9xPgD/npJ2GAa+icOvlAkv3a4y1JfnX2uzN7SQ5JcvLE+0g7Ynwb8Od9rL4oajLUgUDntsofA+vpHK7/uLVN5z/Q+YY9Tee87pU9+myic5HopW9eVb1I55zfMXTu3vkB8AU6t3gOXJJXA/+azvge2dOpkB6GsiZN0Xkj3wE8DvwBnQvDm/tYdyjrUlXPVNUjEw/gR8CzVTXex+rnAp9I8jSdi+ZX9ehzOZ2avPS7DVX1NHASnesofwc8AnwWePmsdmbu7EfnBpWJi8q/R+fC8Pf7WHdR1CRVC+0IbHFI8jY637hXl0UErMlUrMtkSc4G1lXVrw16LAvFQqjJsB8h7BPp3KHzYeAL/oB3WJPerMtkSf4RnU/MGwY9loViodTEQJihdu7uCeAI4HMDHs6CYE16sy6Ttesz43Tuw//ygIezICykmnjKSJIEeIQgSWoW7e8hHHbYYbVy5cpBD2OfuvXWW39QVcv67T8KNYGZ1cWaTGZNehuFukxXk0UbCCtXrmTbtm2DHsY+leShmfQfhZrAzOpiTSazJr2NQl2mq4mnjCRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEnAEATCyvXXsnL9tYMexoJiTXqzJpP5WplslGuy6ANBkjQ3+gqEJA8muSPJ95Jsa22vSrIlyX3t66GtPUkuSjKW5PYkb+7aztrW/74ka7vaj23bH2vrZq53VJK0ZzM5QviNqjqmqla3+fXADVW1is4/I1/f2k8BVrXHOjr/1J0krwIuAI4HjgMumAiR1udDXeut2es9kiTtldmcMjqNzj8Pp309vav98uq4CTgkyRHAycCWqtpVVY8DW4A1bdnBVXVT+xeDl3dtS5I0T/oNhAL+T5Jbk6xrbYdX1cNt+hHg8Da9HNjete6O1ran9h092idJsi7JtiTbxsfH+xy6JKkf/f4/hF+rqp1JfgHYkuRvuhdWVSXZ5/+Ls6o20P4J9erVq/3fn5I0h/o6Qqiqne3rY8Cf0bkG8Gg73UP7+ljrvhM4smv1Fa1tT+0rerRLkubRtIGQ5OeSvGJiGjgJuBPYDEzcKbQWuLpNbwbObncbnQA82U4tXQ+clOTQdjH5JOD6tuypJCe0u4vO7tqWJGme9HPK6HDgz9qdoEuBL1fVnyfZClyV5BzgIeCM1v864FRgDHgG+CBAVe1K8klga+v3iara1abPBS4DDgS+2R6SpHk0bSBU1f3AG3u0/xA4sUd7AedNsa2NwMYe7duAN/QxXknSPuJvKkuSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCZhAISZYkuS3JNW3+qCQ3JxlLcmWS/Vv7y9v8WFu+smsbH2vt9yY5uat9TWsbS7J+7nZPktSvmRwhfBi4p2v+s8CFVfUa4HHgnNZ+DvB4a7+w9SPJ0cCZwOuBNcDnW8gsAS4GTgGOBs5qfSVJ86ivQEiyAng38IU2H+AdwFdbl03A6W36tDZPW35i638acEVVPVdVDwBjwHHtMVZV91fV88AVra8kaR71e4TwOeA/Aj9p8z8PPFFVL7T5HcDyNr0c2A7Qlj/Z+r/Uvts6U7VPkmRdkm1Jto2Pj/c5dElSP6YNhCS/CTxWVbfOw3j2qKo2VNXqqlq9bNmyQQ9HkobK0j76vBX47SSnAgcABwN/BBySZGk7ClgB7Gz9dwJHAjuSLAVeCfywq31C9zpTtUuS5sm0RwhV9bGqWlFVK+lcFP5WVb0P+DbwntZtLXB1m97c5mnLv1VV1drPbHchHQWsAm4BtgKr2l1L+7fn2DwneydJ6ls/RwhT+ShwRZJPAbcBl7b2S4EvJhkDdtF5g6eq7kpyFXA38AJwXlW9CJDkfOB6YAmwsarumsW4JEl7YUaBUFU3Aje26fvp3CG0e59ngfdOsf6ngU/3aL8OuG4mY5EkzS1/U1mSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkS0EcgJDkgyS1J/jrJXUn+S2s/KsnNScaSXJlk/9b+8jY/1pav7NrWx1r7vUlO7mpf09rGkqyf+92UJE2nnyOE54B3VNUbgWOANUlOAD4LXFhVrwEeB85p/c8BHm/tF7Z+JDkaOBN4PbAG+HySJUmWABcDpwBHA2e1vpKkeTRtIFTHj9rsfu1RwDuAr7b2TcDpbfq0Nk9bfmKStPYrquq5qnoAGAOOa4+xqrq/qp4Hrmh9JUnzqK9rCO2T/PeAx4AtwN8CT1TVC63LDmB5m14ObAdoy58Efr67fbd1pmqXJM2jvgKhql6sqmOAFXQ+0b9un45qCknWJdmWZNv4+PgghiBJQ2tGdxlV1RPAt4F/BhySZGlbtALY2aZ3AkcCtOWvBH7Y3b7bOlO193r+DVW1uqpWL1u2bCZDlyRNo5+7jJYlOaRNHwi8C7iHTjC8p3VbC1zdpje3edryb1VVtfYz211IRwGrgFuArcCqdtfS/nQuPG+ei52TJPVv6fRdOALY1O4GehlwVVVdk+Ru4IoknwJuAy5t/S8FvphkDNhF5w2eqroryVXA3cALwHlV9SJAkvOB64ElwMaqumvO9lCS1JdpA6Gqbgfe1KP9fjrXE3ZvfxZ47xTb+jTw6R7t1wHX9TFeSdI+4m8qS5IAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSYyBIkgADQZLUGAiSJMBAkCQ1BoIkCTAQJEmNgSBJAgwESVJjIEiSAANBktQYCJIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBElSM20gJDkyybeT3J3kriQfbu2vSrIlyX3t66GtPUkuSjKW5PYkb+7a1trW/74ka7vaj01yR1vnoiTZFzsrSZpaP0cILwD/vqqOBk4AzktyNLAeuKGqVgE3tHmAU4BV7bEOuAQ6AQJcABwPHAdcMBEirc+HutZbM/tdkyTNxLSBUFUPV9VftemngXuA5cBpwKbWbRNweps+Dbi8Om4CDklyBHAysKWqdlXV48AWYE1bdnBV3VRVBVzetS1J0jyZ0TWEJCuBNwE3A4dX1cNt0SPA4W16ObC9a7UdrW1P7Tt6tPd6/nVJtiXZNj4+PpOhS5Km0XcgJDkI+Brwkap6qntZ+2Rfczy2SapqQ1WtrqrVy5Yt29dPJ0kjpa9ASLIfnTD406r6emt+tJ3uoX19rLXvBI7sWn1Fa9tT+4oe7ZKkedTPXUYBLgXuqar/3rVoMzBxp9Ba4Oqu9rPb3UYnAE+2U0vXAyclObRdTD4JuL4teyrJCe25zu7aliRpnizto89bgQ8AdyT5Xmv7feAzwFVJzgEeAs5oy64DTgXGgGeADwJU1a4knwS2tn6fqKpdbfpc4DLgQOCb7SFJmkfTBkJVfQeY6vcCTuzRv4DzptjWRmBjj/ZtwBumG4skad/xN5UlSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAQaCJKkxECRJgIEgSWoMBEkSYCBIkhoDQZIEGAiSpMZAkCQBBoIkqTEQJEmAgSBJagwESRJgIEiSGgNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElqDARJEmAgSJIaA0GSBBgIkqTGQJAkAX0EQpKNSR5LcmdX26uSbElyX/t6aGtPkouSjCW5Pcmbu9ZZ2/rfl2RtV/uxSe5o61yUJHO9k5Kk6fVzhHAZsGa3tvXADVW1CrihzQOcAqxqj3XAJdAJEOAC4HjgOOCCiRBpfT7Utd7uzyVJmgfTBkJV/QWwa7fm04BNbXoTcHpX++XVcRNwSJIjgJOBLVW1q6oeB7YAa9qyg6vqpqoq4PKubUmS5tHeXkM4vKoebtOPAIe36eXA9q5+O1rbntp39GjvKcm6JNuSbBsfH9/LoUuSepn1ReX2yb7mYCz9PNeGqlpdVauXLVs2H08pSSNjbwPh0Xa6h/b1sda+Eziyq9+K1ran9hU92iVJ82xvA2EzMHGn0Frg6q72s9vdRicAT7ZTS9cDJyU5tF1MPgm4vi17KskJ7e6is7u2JUmaR0un65DkK8DbgcOS7KBzt9BngKuSnAM8BJzRul8HnAqMAc8AHwSoql1JPglsbf0+UVUTF6rPpXMn04HAN9tDkjTPpg2EqjprikUn9uhbwHlTbGcjsLFH+zbgDdONQ5K0b/mbypIkwECQJDUGgiQJMBAkSY2BIEkCDARJUmMgSJIAA0GS1BgIkiTAQJAkNQaCJAkwECRJjYEgSQIMBEnqaeX6a1m5/tpBD2NeGQiStAejFAwGgiQJMBAkqS+jcJRgIEiSAANBktQYCJLUp2G/wGwgSBr6Nzr1Z+mgByBp4egOhQc/8+4BjmRhm6jTsNXIIwRJPXnUMHoMBEl7ZDCMDk8ZaaTM5I1t2E4HzNbK9ddak90M26kjjxCkKfjJeDJrMtwMBGkaE2+CvhH+lPX4WcNSC08ZDbGZvkiH5bB3X+pV01Gum6eRhouBoJf0EyD+8E82bOeRZ2rU93/CMNRhaAJhb9/MhuGbOJ8MjantXptRq4M/S4vf0ARCP/b0Zjabc4D+APysPdVylGo1qqeXRv2X2xZzMI5UIOwrc31BaTG+kPo16kcYo/ZmuZjfHGdrMe77ggmEJGuAPwKWAF+oqs8MeEgakFH5ZD1VOI7Cvg7jPk5lMV14XxCBkGQJcDHwLmAHsDXJ5qq6e7Ajk+bfKATiqJ1WXCyBuCACATgOGKuq+wGSXAGcBhgIEqMREhNG4Xbphfr9TFUNegwkeQ+wpqp+p81/ADi+qs7frd86YF2bfS1wL3AY8IN5HO58mNinV1fVsn5XSjIOPMRw1wRmUJeumuy+jWFgTSbbq5rAyPz87LEmC+UIoS9VtQHY0N2WZFtVrR7QkPaJvd2niW+0Nfmp7hf/sNXFmkw2m/3x52fh/OmKncCRXfMrWpskaZ4slEDYCqxKclSS/YEzgc0DHpMkjZQFccqoql5Icj5wPZ3bTjdW1V19rr5h+i6Lzmz3yZrsu20sJNZkMmvSW1/7tCAuKkuSBm+hnDKSJA2YgSBJAhZ5ICRZk+TeJGNJ1g96PLOVZGOSx5LcOYttWJPJ27AmvbczNHWxJr3NuC5VtSgfdC4+/y3wS8D+wF8DRw96XLPcp7cBbwbutCbWZF/VZBjrYk3mpi6L+QjhpT93UVXPAxN/7mLRqqq/AHbNYhPWZDJr0ttQ1cWa9DbTuizmQFgObO+a39HaRpk1mcya9GZdJhv5mizmQJAkzaHFHAj+uYvJrMlk1qQ36zLZyNdkMQeCf+5iMmsymTXpzbpMNvI1WbSBUFUvABN/7uIe4Krq/89dLEhJvgJ8F3htkh1JzpnJ+tZkMmvS27DVxZr0NtO6+KcrJEnAIj5CkCTNLQNBkgQYCJKkxkCQJAEGgiSpMRAkSYCBIElq/j+7tP7zuNjOcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 5 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "input_data = np.random.randn(1000, 100)  # 1000개의 데이터\n",
    "node_num = 100  # 각 은닉층의 노드(뉴런) 수\n",
    "hidden_layer_size = 5  # 은닉층이 5개\n",
    "activations = {}  # 이곳에 활성화 결과를 저장\n",
    "\n",
    "x = input_data\n",
    "\n",
    "for i in range(hidden_layer_size):\n",
    "    if i != 0:\n",
    "        x = activations[i-1]\n",
    "\n",
    "#     w = np.random.randn(node_num, node_num) * 0.01\n",
    "    w = np.random.randn(node_num, node_num) * np.sqrt(1.0 / node_num) # Xavier Glorot\n",
    "#     w = np.random.randn(node_num, node_num) * np.sqrt(2.0 / node_num) # Kaiming He\n",
    "\n",
    "    a = np.dot(x, w)\n",
    "#     z = sigmoid(a)\n",
    "    z = ReLU(a)\n",
    "#     z = tanh(a)\n",
    "\n",
    "    activations[i] = z\n",
    "\n",
    "for i, a in activations.items():\n",
    "    plt.subplot(1, len(activations), i+1)\n",
    "    plt.title(str(i+1) + \"-layer\")\n",
    "    if i != 0:\n",
    "        plt.yticks([], [])\n",
    "    # plt.xlim(0.1, 1)\n",
    "    # plt.ylim(0, 7000)\n",
    "    plt.hist(a.flatten(), 30, range=(0, 1))\n",
    "\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}