test_bs4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# from Stackexchange"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import bs4\n",
    "import requests\n",
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding(\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "url = \"http://www.imdb.com/search/title?sort=num_votes,desc&start=1&title_type=feature&year=1950,2016\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute first the list of genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "list_of_genres = [u'Action', u'Adult', u'Adventure', u'Animation', u'Biography', u'Comedy', \n",
    "                  u'Crime', u'Drama', u'Family', u'Fantasy', u'Film-Noir', u'History', \n",
    "                  u'Horror', u'Music', u'Musical', u'Mystery', u'Romance', u'Sci-Fi', \n",
    "                  u'Sport', u'Thriller', u'War', u'Western', u'Documentary', u'Game-Show', \n",
    "                  u'Talk-Show', u'News', u'Reality-TV']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cols = ['Title','Runtime','Rating','Year','imdbID']\n",
    "cols.extend(list_of_genres)\n",
    "genre2ind = {str(v):k for k,v in enumerate(list_of_genres)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-31-c7488125e786>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0murl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"http://www.imdb.com/search/title?sort=num_votes,desc&start={}&title_type=feature&tok=0d5b&year=1950,2016\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m     \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrequests\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m     \u001b[0mbs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbs4\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/api.pyc\u001b[0m in \u001b[0;36mget\u001b[0;34m(url, params, **kwargs)\u001b[0m\n\u001b[1;32m     65\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0mkwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'allow_redirects'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 67\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'get'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     68\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     69\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/api.pyc\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0;31m# cases, and look like a memory leak in others.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0msessions\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msession\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0murl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0murl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/sessions.pyc\u001b[0m in \u001b[0;36mrequest\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    466\u001b[0m         }\n\u001b[1;32m    467\u001b[0m         \u001b[0msend_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msettings\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 468\u001b[0;31m         \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0msend_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    469\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    470\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mresp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/sessions.pyc\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    574\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    575\u001b[0m         \u001b[0;31m# Send the request\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 576\u001b[0;31m         \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0madapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrequest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    577\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0;31m# Total elapsed time of the request (approximately)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/adapters.pyc\u001b[0m in \u001b[0;36msend\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    374\u001b[0m                     \u001b[0mdecode_content\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m                     \u001b[0mretries\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmax_retries\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m                     \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    377\u001b[0m                 )\n\u001b[1;32m    378\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.pyc\u001b[0m in \u001b[0;36murlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, **response_kw)\u001b[0m\n\u001b[1;32m    557\u001b[0m             httplib_response = self._make_request(conn, method, url,\n\u001b[1;32m    558\u001b[0m                                                   \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout_obj\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 559\u001b[0;31m                                                   body=body, headers=headers)\n\u001b[0m\u001b[1;32m    560\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    561\u001b[0m             \u001b[0;31m# If we're going to release the connection in ``finally:``, then\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/site-packages/requests/packages/urllib3/connectionpool.pyc\u001b[0m in \u001b[0;36m_make_request\u001b[0;34m(self, conn, method, url, timeout, **httplib_request_kw)\u001b[0m\n\u001b[1;32m    374\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    375\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.7, use buffering of HTTP responses\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 376\u001b[0;31m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbuffering\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    377\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# Python 2.6 and older\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    378\u001b[0m                 \u001b[0mhttplib_response\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgetresponse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/httplib.pyc\u001b[0m in \u001b[0;36mgetresponse\u001b[0;34m(self, buffering)\u001b[0m\n\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1136\u001b[0;31m             \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbegin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1137\u001b[0m             \u001b[0;32massert\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwill_close\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0m_UNKNOWN\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__state\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_CS_IDLE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/httplib.pyc\u001b[0m in \u001b[0;36mbegin\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    451\u001b[0m         \u001b[0;31m# read until we get a non-100 response\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    452\u001b[0m         \u001b[0;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 453\u001b[0;31m             \u001b[0mversion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    454\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mstatus\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0mCONTINUE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    455\u001b[0m                 \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/httplib.pyc\u001b[0m in \u001b[0;36m_read_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    407\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_read_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m         \u001b[0;31m# Initialize with Simple-Response defaults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreadline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_MAXLINE\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    410\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0m_MAXLINE\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    411\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mLineTooLong\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"header line\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/massil/anaconda/lib/python2.7/socket.pyc\u001b[0m in \u001b[0;36mreadline\u001b[0;34m(self, size)\u001b[0m\n\u001b[1;32m    478\u001b[0m             \u001b[0;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 480\u001b[0;31m                     \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_rbufsize\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    481\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    482\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mEINTR\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "f = open(\"first_imdb.csv\",\"w\")\n",
    "f.write(\";\".join(cols)+'\\n')\n",
    "\n",
    "start = 1\n",
    "\n",
    "while start < 100000:\n",
    "#while start < 100:\n",
    "\n",
    "    url = \"http://www.imdb.com/search/title?sort=num_votes,desc&start={}&title_type=feature&tok=0d5b&year=1950,2016\".format(start)\n",
    "    \n",
    "    r = requests.get(url)\n",
    "    bs = bs4.BeautifulSoup(r.text)\n",
    "\n",
    "    for movie in bs.findAll('td','title'):\n",
    "        try:\n",
    "            title = movie.find('a').contents[0]\n",
    "            genres = movie.find('span','genre').findAll('a')\n",
    "            genres = [g.contents[0] for g in genres]\n",
    "            rating = movie.find('span','value').contents[0]\n",
    "            runtime = movie.find('span','runtime').contents[0]\n",
    "            year = movie.find('span','year_type').contents[0]\n",
    "            imdbID = movie.find('span','rating-cancel').a['href'].split('/')[2]\n",
    "            bool_genres = [0]*len(list_of_genres)\n",
    "        except:\n",
    "            print(\"movie {} skipped\".format(movie.find('a').contents[0]))\n",
    "            continue\n",
    "            \n",
    "        for g in genres:\n",
    "            bool_genres[genre2ind[g]] = 1\n",
    "        line = [title, runtime, rating, year, imdbID]\n",
    "        line.extend(bool_genres)\n",
    "        try:\n",
    "            line = map(str,line)\n",
    "        except UnicodeEncodeError:\n",
    "            new_title = title.decode('utf8')\n",
    "            line = [new_title] + map(str,line[1:])\n",
    "            print(line)\n",
    "        f.write(\";\".join(line)+'\\n')\n",
    "\n",
    "    start += 50\n",
    "    \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv('first_imdb.csv',sep=';')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tuto Sam & Max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import urllib2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "webpage = urllib2.urlopen('http://en.wikipedia.org/wiki/Main_Page')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(webpage,'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/\n",
      "#mw-head\n",
      "#p-search\n",
      "/wiki/Wikipedia\n",
      "/wiki/Free_content\n",
      "/wiki/Encyclopedia\n",
      "/wiki/Wikipedia:Introduction\n",
      "/wiki/Special:Statistics\n",
      "/wiki/English_language\n",
      "/wiki/Portal:Arts\n",
      "/wiki/Portal:Biography\n",
      "/wiki/Portal:Geography\n",
      "/wiki/Portal:History\n",
      "/wiki/Portal:Mathematics\n",
      "/wiki/Portal:Science\n",
      "/wiki/Portal:Society\n",
      "/wiki/Portal:Technology\n",
      "/wiki/Portal:Contents/Portals\n",
      "/wiki/File:RihannaJingle_Ball.jpg\n",
      "/wiki/Rehab_(Rihanna_song)\n",
      "/wiki/Barbadian_people\n",
      "/wiki/Rihanna\n",
      "/wiki/Good_Girl_Gone_Bad_(Rihanna_album)\n",
      "/wiki/Def_Jam_Recordings\n",
      "/wiki/Contemporary_hit_radio\n",
      "/wiki/CD_single\n",
      "/wiki/Timbaland\n",
      "/wiki/Justin_Timberlake\n",
      "/wiki/FutureSex/LoveShow\n",
      "/wiki/Contemporary_R%26B\n",
      "/wiki/What_Goes_Around..._Comes_Around\n",
      "/wiki/Anthony_Mandler\n",
      "/wiki/Vasquez_Rocks_Park\n",
      "/wiki/Los_Angeles\n",
      "/wiki/Urban_Music_Awards\n",
      "/wiki/Rehab_(Rihanna_song)\n",
      "/wiki/Battle_of_Kalavrye\n",
      "/wiki/2012_Budweiser_Shootout\n",
      "/wiki/Myles_Standish\n",
      "/wiki/Wikipedia:Today%27s_featured_article/February_2016\n",
      "https://lists.wikimedia.org/mailman/listinfo/daily-article-l\n",
      "/wiki/Wikipedia:Featured_articles\n",
      "/wiki/File:Gwendoline_Konie_(1981).jpg\n",
      "/wiki/Zambia\n",
      "/wiki/Gwendoline_Konie\n",
      "/wiki/6888th_Central_Postal_Directory_Battalion\n",
      "/wiki/Ciro_in_Armenia\n",
      "/wiki/Maria_Teresa_Agnesi_Pinottini\n",
      "/wiki/Vernon_Johnson_(American_football)\n",
      "/wiki/Lone_Star_Conference\n",
      "/wiki/Corruption_in_Haiti\n",
      "/wiki/Lesley_Lewis_(art_historian)\n",
      "/wiki/Courtauld_Institute_of_Art\n",
      "/wiki/Operation_Bedrock_(Laos)\n",
      "/wiki/Randy_W._Berry\n",
      "/wiki/Colorado\n",
      "/wiki/United_States_Department_of_State\n",
      "/wiki/Wikipedia:Recent_additions\n",
      "/wiki/Wikipedia:Your_first_article\n",
      "/wiki/Template_talk:Did_you_know\n",
      "/wiki/File:Alejandro_Gonz%C3%A1lez_I%C3%B1%C3%A1rritu_with_a_camera_in_production.jpg\n",
      "/wiki/2016_Ankara_bombing\n",
      "/wiki/Ankara\n",
      "/wiki/The_Revenant_(2015_film)\n",
      "/wiki/BAFTA_Award_for_Best_Film\n",
      "/wiki/BAFTA_Award_for_Best_Direction\n",
      "/wiki/Alejandro_Gonz%C3%A1lez_I%C3%B1%C3%A1rritu\n",
      "/wiki/69th_British_Academy_Film_Awards\n",
      "/wiki/Pope_Francis\n",
      "/wiki/Patriarch_Kirill_of_Moscow\n",
      "/wiki/Joint_Declaration_of_Pope_Francis_and_Patriarch_Kirill\n",
      "/wiki/LIGO\n",
      "/wiki/First_observation_of_gravitational_waves\n",
      "/wiki/Gravitational_wave\n",
      "/wiki/Bad_Aibling_rail_accident\n",
      "/wiki/Head-on_collision\n",
      "/wiki/Bad_Aibling\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Zika_virus_outbreak_(2015%E2%80%93present)\n",
      "/wiki/European_migrant_crisis\n",
      "/wiki/Deaths_in_2016\n",
      "/wiki/Umberto_Eco\n",
      "/wiki/Harper_Lee\n",
      "/wiki/Boutros_Boutros-Ghali\n",
      "/wiki/February_20\n",
      "/wiki/File:Metropolitan_Museum_of_Art.jpg\n",
      "/wiki/1816\n",
      "/wiki/Gioachino_Rossini\n",
      "/wiki/Opera_buffa\n",
      "/wiki/The_Barber_of_Seville\n",
      "/wiki/Teatro_Argentina\n",
      "/wiki/1872\n",
      "/wiki/Metropolitan_Museum_of_Art\n",
      "/wiki/1959\n",
      "/wiki/John_Diefenbaker\n",
      "/wiki/Avro_Canada_CF-105_Arrow\n",
      "/wiki/Interceptor_aircraft\n",
      "/wiki/1988\n",
      "/wiki/Nagorno-Karabakh_Autonomous_Oblast\n",
      "/wiki/Nagorno-Karabakh_War\n",
      "/wiki/2009\n",
      "/wiki/Liberation_Tigers_of_Tamil_Eelam\n",
      "/wiki/2009_suicide_air_raid_on_Colombo\n",
      "/wiki/C-4_(explosive)\n",
      "/wiki/Colombo\n",
      "/wiki/February_19\n",
      "/wiki/February_20\n",
      "/wiki/February_21\n",
      "/wiki/Wikipedia:Selected_anniversaries/February\n",
      "https://lists.wikimedia.org/mailman/listinfo/daily-article-l\n",
      "/wiki/List_of_historical_anniversaries\n",
      "/wiki/Coordinated_Universal_Time\n",
      "//en.wikipedia.org/w/index.php?title=Main_Page&action=purge\n",
      "/wiki/File:Charadrius_mongolus_-_Laem_Pak_Bia.jpg\n",
      "/wiki/Lesser_sand_plover\n",
      "/wiki/Wader\n",
      "/wiki/Plover\n",
      "/wiki/Bird\n",
      "/wiki/Insect\n",
      "/wiki/Crustacean\n",
      "/wiki/Annelid\n",
      "/wiki/User:JJ_Harrison\n",
      "/wiki/Template:POTD/2016-02-19\n",
      "/wiki/Template:POTD/2016-02-18\n",
      "/wiki/Template:POTD/2016-02-17\n",
      "/wiki/Wikipedia:Picture_of_the_day/February_2016\n",
      "/wiki/Wikipedia:Featured_pictures\n",
      "/wiki/Wikipedia:Community_portal\n",
      "/wiki/Wikipedia:Help_desk\n",
      "/wiki/Wikipedia:Local_Embassy\n",
      "/wiki/Wikipedia:Reference_desk\n",
      "/wiki/Wikipedia:News\n",
      "/wiki/Wikipedia:Village_pump\n",
      "/wiki/Wikimedia_Foundation\n",
      "//wikimediafoundation.org/wiki/Our_projects\n",
      "//commons.wikimedia.org/wiki/\n",
      "//commons.wikimedia.org/\n",
      "//www.mediawiki.org/wiki/\n",
      "//mediawiki.org/\n",
      "//meta.wikimedia.org/wiki/\n",
      "//meta.wikimedia.org/\n",
      "//en.wikibooks.org/wiki/\n",
      "//en.wikibooks.org/\n",
      "//www.wikidata.org/wiki/\n",
      "//www.wikidata.org/\n",
      "//en.wikinews.org/wiki/\n",
      "//en.wikinews.org/\n",
      "//en.wikiquote.org/wiki/\n",
      "//en.wikiquote.org/\n",
      "//en.wikisource.org/wiki/\n",
      "//en.wikisource.org/\n",
      "//species.wikimedia.org/wiki/\n",
      "//species.wikimedia.org/\n",
      "//en.wikiversity.org/wiki/\n",
      "//en.wikiversity.org/\n",
      "//en.wikivoyage.org/wiki/\n",
      "//en.wikivoyage.org/\n",
      "//en.wiktionary.org/wiki/\n",
      "//en.wiktionary.org/\n",
      "/wiki/English_language\n",
      "/wiki/Special:Statistics\n",
      "//de.wikipedia.org/wiki/\n",
      "//es.wikipedia.org/wiki/\n",
      "//fr.wikipedia.org/wiki/\n",
      "//it.wikipedia.org/wiki/\n",
      "//nl.wikipedia.org/wiki/\n",
      "//ja.wikipedia.org/wiki/\n",
      "//pl.wikipedia.org/wiki/\n",
      "//ru.wikipedia.org/wiki/\n",
      "//sv.wikipedia.org/wiki/\n",
      "//vi.wikipedia.org/wiki/\n",
      "//ar.wikipedia.org/wiki/\n",
      "//id.wikipedia.org/wiki/\n",
      "//ms.wikipedia.org/wiki/\n",
      "//ca.wikipedia.org/wiki/\n",
      "//cs.wikipedia.org/wiki/\n",
      "//fa.wikipedia.org/wiki/\n",
      "//ko.wikipedia.org/wiki/\n",
      "//hu.wikipedia.org/wiki/\n",
      "//no.wikipedia.org/wiki/\n",
      "//pt.wikipedia.org/wiki/\n",
      "//ro.wikipedia.org/wiki/\n",
      "//sr.wikipedia.org/wiki/\n",
      "//sh.wikipedia.org/wiki/\n",
      "//fi.wikipedia.org/wiki/\n",
      "//tr.wikipedia.org/wiki/\n",
      "//uk.wikipedia.org/wiki/\n",
      "//zh.wikipedia.org/wiki/\n",
      "//bs.wikipedia.org/wiki/\n",
      "//bg.wikipedia.org/wiki/\n",
      "//da.wikipedia.org/wiki/\n",
      "//et.wikipedia.org/wiki/\n",
      "//el.wikipedia.org/wiki/\n",
      "//simple.wikipedia.org/wiki/\n",
      "//eo.wikipedia.org/wiki/\n",
      "//eu.wikipedia.org/wiki/\n",
      "//gl.wikipedia.org/wiki/\n",
      "//he.wikipedia.org/wiki/\n",
      "//hr.wikipedia.org/wiki/\n",
      "//lv.wikipedia.org/wiki/\n",
      "//lt.wikipedia.org/wiki/\n",
      "//nn.wikipedia.org/wiki/\n",
      "//sk.wikipedia.org/wiki/\n",
      "//sl.wikipedia.org/wiki/\n",
      "//th.wikipedia.org/wiki/\n",
      "//meta.wikimedia.org/wiki/List_of_Wikipedias\n",
      "https://en.wikipedia.org/w/index.php?title=Main_Page&oldid=696846920\n",
      "/wiki/Special:MyTalk\n",
      "/wiki/Special:MyContributions\n",
      "/w/index.php?title=Special:UserLogin&returnto=Main+Page&type=signup\n",
      "/w/index.php?title=Special:UserLogin&returnto=Main+Page\n",
      "/wiki/Main_Page\n",
      "/wiki/Talk:Main_Page\n",
      "#\n",
      "/wiki/Main_Page\n",
      "/w/index.php?title=Main_Page&action=edit\n",
      "/w/index.php?title=Main_Page&action=history\n",
      "#\n",
      "/wiki/Main_Page\n",
      "/wiki/Main_Page\n",
      "/wiki/Portal:Contents\n",
      "/wiki/Portal:Featured_content\n",
      "/wiki/Portal:Current_events\n",
      "/wiki/Special:Random\n",
      "https://donate.wikimedia.org/wiki/Special:FundraiserRedirector?utm_source=donate&utm_medium=sidebar&utm_campaign=C13_en.wikipedia.org&uselang=en\n",
      "//shop.wikimedia.org\n",
      "/wiki/Help:Contents\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:Community_portal\n",
      "/wiki/Special:RecentChanges\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "/wiki/Special:WhatLinksHere/Main_Page\n",
      "/wiki/Special:RecentChangesLinked/Main_Page\n",
      "/wiki/Wikipedia:File_Upload_Wizard\n",
      "/wiki/Special:SpecialPages\n",
      "/w/index.php?title=Main_Page&oldid=696846920\n",
      "/w/index.php?title=Main_Page&action=info\n",
      "//www.wikidata.org/wiki/Q5296\n",
      "/w/index.php?title=Special:CiteThisPage&page=Main_Page&id=696846920\n",
      "/w/index.php?title=Special:Book&bookcmd=book_creator&referer=Main+Page\n",
      "/w/index.php?title=Special:Book&bookcmd=render_article&arttitle=Main+Page&returnto=Main+Page&oldid=696846920&writer=rdf2latex\n",
      "/w/index.php?title=Main_Page&printable=yes\n",
      "https://commons.wikimedia.org/wiki/Main_Page\n",
      "https://meta.wikimedia.org/wiki/Main_Page\n",
      "https://species.wikimedia.org/wiki/Main_Page\n",
      "https://en.wikibooks.org/wiki/Main_Page\n",
      "https://www.wikidata.org/wiki/Wikidata:Main_Page\n",
      "https://en.wikinews.org/wiki/Main_Page\n",
      "https://en.wikiquote.org/wiki/Main_Page\n",
      "https://en.wikisource.org/wiki/Main_Page\n",
      "https://en.wikivoyage.org/wiki/Main_Page\n",
      "//simple.wikipedia.org/wiki/\n",
      "//ar.wikipedia.org/wiki/\n",
      "//id.wikipedia.org/wiki/\n",
      "//ms.wikipedia.org/wiki/\n",
      "//bs.wikipedia.org/wiki/\n",
      "//bg.wikipedia.org/wiki/\n",
      "//ca.wikipedia.org/wiki/\n",
      "//cs.wikipedia.org/wiki/\n",
      "//da.wikipedia.org/wiki/\n",
      "//de.wikipedia.org/wiki/\n",
      "//et.wikipedia.org/wiki/\n",
      "//el.wikipedia.org/wiki/\n",
      "//es.wikipedia.org/wiki/\n",
      "//eo.wikipedia.org/wiki/\n",
      "//eu.wikipedia.org/wiki/\n",
      "//fa.wikipedia.org/wiki/\n",
      "//fr.wikipedia.org/wiki/\n",
      "//gl.wikipedia.org/wiki/\n",
      "//ko.wikipedia.org/wiki/\n",
      "//he.wikipedia.org/wiki/\n",
      "//hr.wikipedia.org/wiki/\n",
      "//it.wikipedia.org/wiki/\n",
      "//ka.wikipedia.org/wiki/\n",
      "//lv.wikipedia.org/wiki/\n",
      "//lt.wikipedia.org/wiki/\n",
      "//hu.wikipedia.org/wiki/\n",
      "//nl.wikipedia.org/wiki/\n",
      "//ja.wikipedia.org/wiki/\n",
      "//no.wikipedia.org/wiki/\n",
      "//nn.wikipedia.org/wiki/\n",
      "//pl.wikipedia.org/wiki/\n",
      "//pt.wikipedia.org/wiki/\n",
      "//ro.wikipedia.org/wiki/\n",
      "//ru.wikipedia.org/wiki/\n",
      "//sk.wikipedia.org/wiki/\n",
      "//sl.wikipedia.org/wiki/\n",
      "//sr.wikipedia.org/wiki/\n",
      "//sh.wikipedia.org/wiki/\n",
      "//fi.wikipedia.org/wiki/\n",
      "//sv.wikipedia.org/wiki/\n",
      "//th.wikipedia.org/wiki/\n",
      "//vi.wikipedia.org/wiki/\n",
      "//tr.wikipedia.org/wiki/\n",
      "//uk.wikipedia.org/wiki/\n",
      "//zh.wikipedia.org/wiki/\n",
      "#\n",
      "//en.wikipedia.org/wiki/Wikipedia:Text_of_Creative_Commons_Attribution-ShareAlike_3.0_Unported_License\n",
      "//creativecommons.org/licenses/by-sa/3.0/\n",
      "//wikimediafoundation.org/wiki/Terms_of_Use\n",
      "//wikimediafoundation.org/wiki/Privacy_policy\n",
      "//www.wikimediafoundation.org/\n",
      "//wikimediafoundation.org/wiki/Privacy_policy\n",
      "/wiki/Wikipedia:About\n",
      "/wiki/Wikipedia:General_disclaimer\n",
      "//en.wikipedia.org/wiki/Wikipedia:Contact_us\n",
      "https://www.mediawiki.org/wiki/Special:MyLanguage/How_to_contribute\n",
      "//wikimediafoundation.org/wiki/Cookie_statement\n",
      "//en.m.wikipedia.org/w/index.php?title=Main_Page&mobileaction=toggle_view_mobile\n",
      "//wikimediafoundation.org/\n",
      "//www.mediawiki.org/\n"
     ]
    }
   ],
   "source": [
    "for anchor in soup.find_all('a'):\n",
    "    print(anchor.get('href', '/'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}