scrape_pokemon_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import lxml.html as lh\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#url='http://pokemondb.net/pokedex/all'\n",
    "#url = 'https://pokemondb.net/pokedex/game/sword-shield'\n",
    "url = 'https://www.eurogamer.net/articles/2019-10-31-pokemon-sword-shield-galar-pokedex-6017'\n",
    "page = requests.get(url)\n",
    "doc = lh.fromstring(page.content)\n",
    "#span_elements = doc.xpath('//span')\n",
    "tr_elements = doc.xpath('//tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(T) for T in tr_elements[:12]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Pokedex #', []),\n",
       " ('Pokemon', []),\n",
       " ('Type', []),\n",
       " ('', []),\n",
       " ('Generation', [])]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = []\n",
    "col_names = [\"Pokedex #\", \"Pokemon\", \"Type\", \"\", \"Generation\"]\n",
    "for t in col_names:\n",
    "    name= t\n",
    "#    print('%d: \"%s\"'%(i,name))\n",
    "    col.append((name,[]))\n",
    "col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(0, len(tr_elements)):\n",
    "#for j in range(1, len(span_elements)):\n",
    "    T = tr_elements[j]\n",
    "#    T = span_elements[j]\n",
    "    if len(T) != 5:\n",
    "        pass\n",
    "    \n",
    "    i = 0\n",
    "    \n",
    "    for t in T.iterchildren():\n",
    "        data=t.text_content()\n",
    "        if i > 0:\n",
    "            try:\n",
    "                data = int(data)\n",
    "            except:\n",
    "                pass\n",
    "        col[i][1].append(data)\n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[407, 407, 407, 407, 407]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(C) for (title,C) in col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dict={title:column for (title,column) in col}\n",
    "df=pd.DataFrame(Dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df[\"Pokedex #\"] != 'TBC']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['Pokedex #']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df['Pokedex #'])):\n",
    "    x = df['Pokedex #'].iloc[i]\n",
    "    x = x.split('#')\n",
    "    if len(x) > 0:\n",
    "        df['Pokedex #'].iloc[i] = x[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pokemon</th>\n",
       "      <th>Type</th>\n",
       "      <th></th>\n",
       "      <th>Generation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pokedex #</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Grookey</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Thwackey</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Rillaboom</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Scorbunny</td>\n",
       "      <td>Fire</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Raboot</td>\n",
       "      <td>Fire</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>Drakloak</td>\n",
       "      <td>Dragon / Ghost</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>Dragapult</td>\n",
       "      <td>Dragon / Ghost</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>Zacian</td>\n",
       "      <td>Fairy (Fairy / Steel)</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>Zamazenta</td>\n",
       "      <td>Fighting (Fighting / Steel)</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>Eternatus</td>\n",
       "      <td>Poison / Dragon</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             Pokemon                         Type       Generation\n",
       "Pokedex #                                                         \n",
       "1           Grookey                         Grass     Gen 8 (SWSH)\n",
       "2           Thwackey                       Grass      Gen 8 (SWSH)\n",
       "3          Rillaboom                        Grass     Gen 8 (SWSH)\n",
       "4          Scorbunny                         Fire     Gen 8 (SWSH)\n",
       "5             Raboot                         Fire     Gen 8 (SWSH)\n",
       "...              ...                          ... ..           ...\n",
       "396         Drakloak               Dragon / Ghost     Gen 8 (SWSH)\n",
       "397        Dragapult               Dragon / Ghost     Gen 8 (SWSH)\n",
       "398           Zacian        Fairy (Fairy / Steel)     Gen 8 (SWSH)\n",
       "399        Zamazenta  Fighting (Fighting / Steel)     Gen 8 (SWSH)\n",
       "400        Eternatus              Poison / Dragon     Gen 8 (SWSH)\n",
       "\n",
       "[400 rows x 4 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.set_index('Pokedex #')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://www.dexerto.com/pokemon/pokemon-sword-and-shield-spawn-locations-guide-1287933'\n",
    "\n",
    "page = requests.get(url)\n",
    "doc = lh.fromstring(page.content)\n",
    "tr_elements = doc.xpath('//tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[4, 3, 4]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[len(T) for T in tr_elements[247:250]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "del tr_elements[248]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1: \"Pokedex #\"\n",
      "2: \"Pokemon\"\n",
      "3: \"Overworld\"\n",
      "4: \"Non-Overworld\"\n"
     ]
    }
   ],
   "source": [
    "col = []\n",
    "i = 0\n",
    "\n",
    "for t in tr_elements[0]:\n",
    "    i+=1\n",
    "    name=t.text_content()\n",
    "    print('%d: \"%s\"'%(i,name))\n",
    "    col.append((name,[]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(1, len(tr_elements)):\n",
    "    T = tr_elements[j]\n",
    "    if len(T) != 4:\n",
    "        break    \n",
    "    i = 0\n",
    "    \n",
    "    for t in T.iterchildren():\n",
    "        data=t.text_content()\n",
    "        if i > 0:\n",
    "            try:\n",
    "                data = int(data)\n",
    "            except:\n",
    "                pass\n",
    "        col[i][1].append(data)\n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dict={title:column for (title,column) in col}\n",
    "df_location=pd.DataFrame(Dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pokedex #</th>\n",
       "      <th>Pokemon</th>\n",
       "      <th>Overworld</th>\n",
       "      <th>Non-Overworld</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Grookey</td>\n",
       "      <td>Starter</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Thwackey</td>\n",
       "      <td>Evolve Grookey</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Rillaboom</td>\n",
       "      <td>Evolve Thwacket</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Scorbunny</td>\n",
       "      <td>Starter</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Raboot</td>\n",
       "      <td>Evolve Scorbunny</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Pokedex #    Pokemon         Overworld Non-Overworld\n",
       "0         1    Grookey           Starter              \n",
       "1         2   Thwackey    Evolve Grookey              \n",
       "2         3  Rillaboom   Evolve Thwacket              \n",
       "3         4  Scorbunny           Starter              \n",
       "4         5     Raboot  Evolve Scorbunny              "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_location.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pokedex #</th>\n",
       "      <th>Pokemon</th>\n",
       "      <th>Overworld</th>\n",
       "      <th>Non-Overworld</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>248</td>\n",
       "      <td>Throh</td>\n",
       "      <td>Motostoke Outskirts, Bridge Field, Giant's Mirror</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pokedex # Pokemon                                          Overworld  \\\n",
       "0        248   Throh  Motostoke Outskirts, Bridge Field, Giant's Mirror   \n",
       "\n",
       "  Non-Overworld  \n",
       "0                "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d = {'Pokedex #': [248], 'Pokemon': ['Throh'], 'Overworld': [\"Motostoke Outskirts, Bridge Field, Giant's Mirror\"], 'Non-Overworld': [\"\"]}\n",
    "throh_df = pd.DataFrame(data=d)\n",
    "throh_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "location_df = df_location.append(throh_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "location_df['Pokedex #'] = location_df['Pokedex #'].astype(int)\n",
    "location_df = location_df.set_index('Pokedex #')\n",
    "location_df.sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "You are trying to merge on object and int64 columns. If you wish to proceed you should use pd.concat",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-b39727479f8f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpokemon_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlocation_df\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Pokedex #'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib64/python3.7/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(self, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m   7295\u001b[0m             \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   7296\u001b[0m             \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 7297\u001b[0;31m             \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   7298\u001b[0m         )\n\u001b[1;32m   7299\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib64/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36mmerge\u001b[0;34m(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m     84\u001b[0m         \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m         \u001b[0mindicator\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mindicator\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m         \u001b[0mvalidate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m     )\n\u001b[1;32m     88\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib64/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, left, right, how, on, left_on, right_on, axis, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[1;32m    629\u001b[0m         \u001b[0;31m# validate the merge keys dtypes. We may need to coerce\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m         \u001b[0;31m# to avoid incompat dtypes\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 631\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_coerce_merge_keys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    632\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    633\u001b[0m         \u001b[0;31m# If argument passed to validate,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib64/python3.7/site-packages/pandas/core/reshape/merge.py\u001b[0m in \u001b[0;36m_maybe_coerce_merge_keys\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1144\u001b[0m                     \u001b[0minferred_right\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstring_types\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0minferred_left\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstring_types\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1145\u001b[0m                 ):\n\u001b[0;32m-> 1146\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1148\u001b[0m             \u001b[0;31m# datetimelikes must match exactly\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: You are trying to merge on object and int64 columns. If you wish to proceed you should use pd.concat"
     ]
    }
   ],
   "source": [
    "pokemon_df = df.merge(location_df, on='Pokedex #')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "pokemon_df = pd.concat([df, location_df], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pokedex #</th>\n",
       "      <th>Pokemon</th>\n",
       "      <th>Type</th>\n",
       "      <th></th>\n",
       "      <th>Generation</th>\n",
       "      <th>Pokemon</th>\n",
       "      <th>Overworld</th>\n",
       "      <th>Non-Overworld</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Grookey</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Thwackey</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Grookey</td>\n",
       "      <td>Starter</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Rillaboom</td>\n",
       "      <td>Grass</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Thwackey</td>\n",
       "      <td>Evolve Grookey</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Scorbunny</td>\n",
       "      <td>Fire</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Rillaboom</td>\n",
       "      <td>Evolve Thwacket</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Raboot</td>\n",
       "      <td>Fire</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Scorbunny</td>\n",
       "      <td>Starter</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>397</td>\n",
       "      <td>Dragapult</td>\n",
       "      <td>Dragon / Ghost</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Drakloak</td>\n",
       "      <td>Lake of Outrage</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>398</td>\n",
       "      <td>Zacian</td>\n",
       "      <td>Fairy (Fairy / Steel)</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Dragapult</td>\n",
       "      <td>Evolve Drakloak</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>399</td>\n",
       "      <td>Zamazenta</td>\n",
       "      <td>Fighting (Fighting / Steel)</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Zacian</td>\n",
       "      <td>Legendary</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>400</td>\n",
       "      <td>Eternatus</td>\n",
       "      <td>Poison / Dragon</td>\n",
       "      <td></td>\n",
       "      <td>Gen 8 (SWSH)</td>\n",
       "      <td>Zamazenta</td>\n",
       "      <td>Legendary</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Eternatus</td>\n",
       "      <td>Legendary</td>\n",
       "      <td>-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>401 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Pokedex #    Pokemon                         Type         Generation  \\\n",
       "0           1   Grookey                         Grass       Gen 8 (SWSH)   \n",
       "1           2   Thwackey                       Grass        Gen 8 (SWSH)   \n",
       "2           3  Rillaboom                        Grass       Gen 8 (SWSH)   \n",
       "3           4  Scorbunny                         Fire       Gen 8 (SWSH)   \n",
       "4           5     Raboot                         Fire       Gen 8 (SWSH)   \n",
       "..        ...        ...                          ...  ...           ...   \n",
       "396       397  Dragapult               Dragon / Ghost       Gen 8 (SWSH)   \n",
       "397       398     Zacian        Fairy (Fairy / Steel)       Gen 8 (SWSH)   \n",
       "398       399  Zamazenta  Fighting (Fighting / Steel)       Gen 8 (SWSH)   \n",
       "399       400  Eternatus              Poison / Dragon       Gen 8 (SWSH)   \n",
       "400       NaN        NaN                          NaN  NaN           NaN   \n",
       "\n",
       "       Pokemon        Overworld Non-Overworld  \n",
       "0          NaN              NaN           NaN  \n",
       "1      Grookey          Starter                \n",
       "2     Thwackey   Evolve Grookey                \n",
       "3    Rillaboom  Evolve Thwacket                \n",
       "4    Scorbunny          Starter                \n",
       "..         ...              ...           ...  \n",
       "396   Drakloak  Lake of Outrage             -  \n",
       "397  Dragapult  Evolve Drakloak             -  \n",
       "398     Zacian        Legendary             -  \n",
       "399  Zamazenta        Legendary             -  \n",
       "400  Eternatus        Legendary             -  \n",
       "\n",
       "[401 rows x 8 columns]"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pokemon_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "url='https://www.polygon.com/pokemon-sword-shield-guide/2019/11/16/20968169/type-strength-weakness-super-effective-weakness-chart'\n",
    "\n",
    "page = requests.get(url)\n",
    "doc = lh.fromstring(page.content)\n",
    "tr_elements = doc.xpath('//tr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "[len(T) for T in tr_elements[:12]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col = []\n",
    "i = 0\n",
    "\n",
    "for t in tr_elements[0]:\n",
    "    i+=1\n",
    "    name=t.text_content()\n",
    "    print('%d: \"%s\"'%(i,name))\n",
    "    col.append((name,[]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for j in range(1, len(tr_elements)):\n",
    "    T = tr_elements[j]\n",
    "    if len(T) != 3:\n",
    "        break    \n",
    "    i = 0\n",
    "    \n",
    "    for t in T.iterchildren():\n",
    "        data=t.text_content()\n",
    "        if i > 0:\n",
    "            try:\n",
    "                data = int(data)\n",
    "            except:\n",
    "                pass\n",
    "        col[i][1].append(data)\n",
    "        i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Dict={title:column for (title,column) in col}\n",
    "df_battle=pd.DataFrame(Dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_battle = df_battle.drop(0)\n",
    "df_battle = df_battle.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_battle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pokemon_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}