notebooks/fake-news-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of fake news sites and viral posts, 2016 vs. 2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import calendar\n",
    "from datetime import datetime\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing lists of sites from 2016 vs. 2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load 2017 list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>network</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12minutos.com</td>\n",
       "      <td>React365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20minutenews.com</td>\n",
       "      <td>Celebtricity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24aktuelles.com</td>\n",
       "      <td>React365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>24x365live.com</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>abcnews-us.com</td>\n",
       "      <td>Batty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             domain       network\n",
       "0     12minutos.com      React365\n",
       "1  20minutenews.com  Celebtricity\n",
       "2   24aktuelles.com      React365\n",
       "3    24x365live.com           NaN\n",
       "4    abcnews-us.com         Batty"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2017 = pd.read_csv(\"../data/sites_2017.csv\")\n",
    "    \n",
    "sites_2017.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "167"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sites_2017)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Subdomained sites:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "      <th>network</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>abcnews.com.co</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>cnn.com.de</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>nachrichten.de.com</td>\n",
       "      <td>React365</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                domain   network\n",
       "5       abcnews.com.co       NaN\n",
       "48          cnn.com.de       NaN\n",
       "83  nachrichten.de.com  React365"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2017[\n",
    "    sites_2017[\"domain\"].str.count(r\"\\.\") > 1\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No sites show up twice:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2017[\"domain\"].value_counts().max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load 2016 list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abcnews.com.co</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>adobochronicles.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>alynews.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>associatedmediacoverage.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>baldwinpost.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        domain\n",
       "0               abcnews.com.co\n",
       "1          adobochronicles.com\n",
       "2                  alynews.com\n",
       "3  associatedmediacoverage.com\n",
       "4              baldwinpost.com"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2016 = pd.read_csv(\"../data/sites_2016.csv\")\n",
    "\n",
    "sites_2016.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sites_2016)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Subdomained sites:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>domain</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>abcnews.com.co</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           domain\n",
       "0  abcnews.com.co"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2016[\n",
    "    sites_2016[\"domain\"].str.count(r\"\\.\") > 1\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No sites show up twice:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sites_2016[\"domain\"].value_counts().max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sites in 2017 but not 2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "119"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(sites_2017[\"domain\"]) - set(sites_2016[\"domain\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sites in 2016 but not 2017"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(sites_2016[\"domain\"]) - set(sites_2017[\"domain\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Percentage of 2016 list that are not in 2017 list:\n",
    "100 * len(set(sites_2016[\"domain\"]) - set(sites_2017[\"domain\"]))  / len(set(sites_2016[\"domain\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sites in both lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "48"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(sites_2016[\"domain\"]).intersection(set(sites_2017[\"domain\"])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Top domains in 2017's top 50 posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>domain</th>\n",
       "      <th>fb_engagements</th>\n",
       "      <th>published_date</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Babysitter transported to hospital after inser...</td>\n",
       "      <td>http://worldnewsdailyreport.com/babysitter-tra...</td>\n",
       "      <td>worldnewsdailyreport.com</td>\n",
       "      <td>1204400</td>\n",
       "      <td>2017-05-03</td>\n",
       "      <td>Medical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FBI seizes over 3,000 penises during raid at m...</td>\n",
       "      <td>http://worldnewsdailyreport.com/fbi-seizes-ove...</td>\n",
       "      <td>worldnewsdailyreport.com</td>\n",
       "      <td>1145470</td>\n",
       "      <td>2017-09-25</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Charles Manson to be released on parole, to Jo...</td>\n",
       "      <td>http://www.breakingnews365.net/59690fb994b9c/c...</td>\n",
       "      <td>breakingnews365.net</td>\n",
       "      <td>1125031</td>\n",
       "      <td>2017-07-14</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Police: Chester Bennington Was Murdered</td>\n",
       "      <td>http://yournewswire.com/chester-bennington-mur...</td>\n",
       "      <td>yournewswire.com</td>\n",
       "      <td>1042032</td>\n",
       "      <td>2017-07-21</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Morgue employee cremated by mistake while taki...</td>\n",
       "      <td>https://worldnewsdailyreport.com/morgue-employ...</td>\n",
       "      <td>worldnewsdailyreport.com</td>\n",
       "      <td>993205</td>\n",
       "      <td>2017-03-01</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  Babysitter transported to hospital after inser...   \n",
       "1  FBI seizes over 3,000 penises during raid at m...   \n",
       "2  Charles Manson to be released on parole, to Jo...   \n",
       "3            Police: Chester Bennington Was Murdered   \n",
       "4  Morgue employee cremated by mistake while taki...   \n",
       "\n",
       "                                                 url  \\\n",
       "0  http://worldnewsdailyreport.com/babysitter-tra...   \n",
       "1  http://worldnewsdailyreport.com/fbi-seizes-ove...   \n",
       "2  http://www.breakingnews365.net/59690fb994b9c/c...   \n",
       "3  http://yournewswire.com/chester-bennington-mur...   \n",
       "4  https://worldnewsdailyreport.com/morgue-employ...   \n",
       "\n",
       "                     domain  fb_engagements published_date category  \n",
       "0  worldnewsdailyreport.com         1204400     2017-05-03  Medical  \n",
       "1  worldnewsdailyreport.com         1145470     2017-09-25    Crime  \n",
       "2       breakingnews365.net         1125031     2017-07-14    Crime  \n",
       "3          yournewswire.com         1042032     2017-07-21    Crime  \n",
       "4  worldnewsdailyreport.com          993205     2017-03-01    Crime  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the top 50 list:\n",
    "top_2017 = pd.read_csv(\"../data/top_2017.csv\")\n",
    "\n",
    "top_2017.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>worldnewsdailyreport.com</th>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yournewswire.com</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel45news.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>breakingnews365.net</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>breakingnews247.net</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viralactions.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12minutos.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          count\n",
       "worldnewsdailyreport.com     12\n",
       "yournewswire.com              9\n",
       "channel45news.com             2\n",
       "breakingnews365.net           2\n",
       "breakingnews247.net           2\n",
       "viralactions.com              2\n",
       "12minutos.com                 2"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2017_domains = top_2017[\"domain\"].value_counts()\\\n",
    "    .to_frame(\"count\")\n",
    "\n",
    "top_2017_domains[\n",
    "    top_2017_domains[\"count\"] > 1\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What happened to the top 2016 publishers in 2017?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First let's figure out who the top publishers were:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>domain</th>\n",
       "      <th>month</th>\n",
       "      <th>fb_engagements</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\"Obama Signs Executive Order Banning The Pledg...</td>\n",
       "      <td>http://abcnews.com.co/obama-executive-order-ba...</td>\n",
       "      <td>abcnews.com.co</td>\n",
       "      <td>October</td>\n",
       "      <td>2176777</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Woman arrested for defecating on boss’ desk a...</td>\n",
       "      <td>https://thevalleyreport.com/2016/04/25/woman-a...</td>\n",
       "      <td>thevalleyreport.com</td>\n",
       "      <td>April</td>\n",
       "      <td>1765146</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"Pope Francis Shocks World, Endorses Donald Tr...</td>\n",
       "      <td>Ending The Fed (removed)</td>\n",
       "      <td>ending the fed (removed)</td>\n",
       "      <td>October</td>\n",
       "      <td>961000</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"Trump Offering Free One-Way Tickets to Africa...</td>\n",
       "      <td>http://tmzhiphop.com/trump-offering-free-one-w...</td>\n",
       "      <td>tmzhiphop.com</td>\n",
       "      <td>November</td>\n",
       "      <td>801741</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"Cinnamon Roll Can Explodes Inside Man’s Butt ...</td>\n",
       "      <td>https://empireherald.com/cinnamon-roll-can-exp...</td>\n",
       "      <td>empireherald.com</td>\n",
       "      <td>April</td>\n",
       "      <td>764814</td>\n",
       "      <td>Crime</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title  \\\n",
       "0  \"Obama Signs Executive Order Banning The Pledg...   \n",
       "1  \"Woman arrested for defecating on boss’ desk a...   \n",
       "2  \"Pope Francis Shocks World, Endorses Donald Tr...   \n",
       "3  \"Trump Offering Free One-Way Tickets to Africa...   \n",
       "4  \"Cinnamon Roll Can Explodes Inside Man’s Butt ...   \n",
       "\n",
       "                                                 url  \\\n",
       "0  http://abcnews.com.co/obama-executive-order-ba...   \n",
       "1  https://thevalleyreport.com/2016/04/25/woman-a...   \n",
       "2                           Ending The Fed (removed)   \n",
       "3  http://tmzhiphop.com/trump-offering-free-one-w...   \n",
       "4  https://empireherald.com/cinnamon-roll-can-exp...   \n",
       "\n",
       "                     domain     month  fb_engagements  category  \n",
       "0            abcnews.com.co   October         2176777  Politics  \n",
       "1       thevalleyreport.com     April         1765146     Crime  \n",
       "2  ending the fed (removed)   October          961000  Politics  \n",
       "3             tmzhiphop.com  November          801741  Politics  \n",
       "4          empireherald.com     April          764814     Crime  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2016 = pd.read_csv('../data/top_2016.csv')\n",
    "\n",
    "top_2016.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "URLs for three posts are no longer available:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>url</th>\n",
       "      <th>domain</th>\n",
       "      <th>month</th>\n",
       "      <th>fb_engagements</th>\n",
       "      <th>category</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\"Pope Francis Shocks World, Endorses Donald Tr...</td>\n",
       "      <td>Ending The Fed (removed)</td>\n",
       "      <td>ending the fed (removed)</td>\n",
       "      <td>October</td>\n",
       "      <td>961000</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"FBI Agent Suspected in Hillary Email Leaks Fo...</td>\n",
       "      <td>Denver Guardian (removed)</td>\n",
       "      <td>denver guardian (removed)</td>\n",
       "      <td>November</td>\n",
       "      <td>567000</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>BREAKING Romanian Hacker With Access To Clinto...</td>\n",
       "      <td>Christian Times (removed)</td>\n",
       "      <td>christian times (removed)</td>\n",
       "      <td>July</td>\n",
       "      <td>242500</td>\n",
       "      <td>Politics</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                title  \\\n",
       "2   \"Pope Francis Shocks World, Endorses Donald Tr...   \n",
       "6   \"FBI Agent Suspected in Hillary Email Leaks Fo...   \n",
       "37  BREAKING Romanian Hacker With Access To Clinto...   \n",
       "\n",
       "                          url                     domain     month  \\\n",
       "2    Ending The Fed (removed)   ending the fed (removed)   October   \n",
       "6   Denver Guardian (removed)  denver guardian (removed)  November   \n",
       "37  Christian Times (removed)  christian times (removed)      July   \n",
       "\n",
       "    fb_engagements  category  \n",
       "2           961000  Politics  \n",
       "6           567000  Politics  \n",
       "37          242500  Politics  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2016[\n",
    "    top_2016[\"domain\"].str.count(r\"\\.\") == 0\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tmzhiphop.com</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abcnews.com.co</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldnewsdailyreport.com</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thevalleyreport.com</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>empireherald.com</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>burrardstreetjournal.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>celebtricity.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dailyfinesser.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>satiratribune.com</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heaviermetal.net</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thelastlineofdefense.org</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          count\n",
       "tmzhiphop.com                 7\n",
       "abcnews.com.co                6\n",
       "worldnewsdailyreport.com      5\n",
       "thevalleyreport.com           4\n",
       "empireherald.com              3\n",
       "burrardstreetjournal.com      2\n",
       "celebtricity.com              2\n",
       "dailyfinesser.com             2\n",
       "satiratribune.com             2\n",
       "heaviermetal.net              2\n",
       "thelastlineofdefense.org      2"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2016_domains = top_2016[\"domain\"].value_counts()\\\n",
    "    .to_frame(\"count\")\n",
    "\n",
    "top_2016_domains[\n",
    "    top_2016_domains[\"count\"] > 1\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(top_2016_domains)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count_2016</th>\n",
       "      <th>count_2017</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>tmzhiphop.com</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abcnews.com.co</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>worldnewsdailyreport.com</th>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thevalleyreport.com</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>empireherald.com</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>burrardstreetjournal.com</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dailyfinesser.com</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thelastlineofdefense.org</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>satiratribune.com</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>celebtricity.com</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>heaviermetal.net</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>usanewsflash.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kypo6.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>politicops.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>react365.com</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>christian times (removed)</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>bizstandardnews.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conservativestate.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>baltimoregazette.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>denver guardian (removed)</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>adobochronicles.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>thebostontribune.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ending the fed (removed)</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>straightstoned.com</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sundayinquirer.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12minutos.com</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tmzbreaking.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viralactions.com</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>viralmugshot.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>theseattletribune.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>empirenews.net</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>redcountry.us</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>realnewsrightnow.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noticias-frescas.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>abcnews-us.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>civictribune.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel45news.com</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel28news.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel24news.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>channel22news.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>breakingnews365.net</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>breakingnews247.net</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>borderherald.com</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actualite.co</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yournewswire.com</th>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           count_2016  count_2017\n",
       "tmzhiphop.com                       7           0\n",
       "abcnews.com.co                      6           0\n",
       "worldnewsdailyreport.com            5          12\n",
       "thevalleyreport.com                 4           0\n",
       "empireherald.com                    3           0\n",
       "burrardstreetjournal.com            2           1\n",
       "dailyfinesser.com                   2           0\n",
       "thelastlineofdefense.org            2           1\n",
       "satiratribune.com                   2           0\n",
       "celebtricity.com                    2           1\n",
       "heaviermetal.net                    2           0\n",
       "usanewsflash.com                    1           0\n",
       "kypo6.com                           1           0\n",
       "politicops.com                      1           0\n",
       "react365.com                        1           1\n",
       "christian times (removed)           1           0\n",
       "bizstandardnews.com                 1           0\n",
       "conservativestate.com               1           0\n",
       "baltimoregazette.com                1           0\n",
       "denver guardian (removed)           1           0\n",
       "adobochronicles.com                 1           0\n",
       "thebostontribune.com                1           0\n",
       "ending the fed (removed)            1           0\n",
       "straightstoned.com                  1           0\n",
       "sundayinquirer.com                  0           1\n",
       "12minutos.com                       0           2\n",
       "tmzbreaking.com                     0           1\n",
       "viralactions.com                    0           2\n",
       "viralmugshot.com                    0           1\n",
       "theseattletribune.com               0           1\n",
       "empirenews.net                      0           1\n",
       "redcountry.us                       0           1\n",
       "realnewsrightnow.com                0           1\n",
       "noticias-frescas.com                0           1\n",
       "abcnews-us.com                      0           1\n",
       "civictribune.com                    0           1\n",
       "channel45news.com                   0           2\n",
       "channel28news.com                   0           1\n",
       "channel24news.com                   0           1\n",
       "channel22news.com                   0           1\n",
       "breakingnews365.net                 0           2\n",
       "breakingnews247.net                 0           2\n",
       "borderherald.com                    0           1\n",
       "actualite.co                        0           1\n",
       "yournewswire.com                    0           9"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_domains_comparison = pd.merge(\n",
    "    top_2016_domains,\n",
    "    top_2017_domains,\n",
    "    how=\"outer\",\n",
    "    left_index=True,\n",
    "    right_index=True,\n",
    "    suffixes=[\"_2016\", \"_2017\"]\n",
    ").fillna(0).astype(int)\n",
    "\n",
    "top_domains_comparison.sort_values(\"count_2016\", ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Write to file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_domains_comparison.sort_values(\"count_2016\", ascending=False)\\\n",
    "    .to_csv(\"../output/top_domains_comparison.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What types of stories did well in 2017?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Crime</th>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Politics</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Medical</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Race</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Breaking News</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Religion</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Local News</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Music</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sports</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               count\n",
       "Crime             20\n",
       "Politics          11\n",
       "World              4\n",
       "Medical            4\n",
       "Business           3\n",
       "Race               2\n",
       "Breaking News      2\n",
       "Religion           1\n",
       "Local News         1\n",
       "Music              1\n",
       "Sports             1"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_cats_2017 = top_2017['category'].fillna(\"other\").value_counts().to_frame(\"count\")\n",
    "top_cats_2017"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Politics and crime dominate the categories. What percentage do they make up of the whole?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# politics\n",
    "top_cats_2017['count']['Politics'] / len(top_2017) * 100\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# crime\n",
    "top_cats_2017['count']['Crime'] / len(top_2017) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How does this compare to 2016?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Politics</th>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Crime</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>World</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Science</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Race</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Religion</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sexuality</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Animals</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Music</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           count\n",
       "Politics      23\n",
       "Crime         16\n",
       "World          3\n",
       "Science        2\n",
       "Race           2\n",
       "Religion       1\n",
       "Sexuality      1\n",
       "Animals        1\n",
       "Music          1"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_cats_2016 = top_2016['category']\\\n",
    "    .value_counts()\\\n",
    "    .to_frame(\"count\")\n",
    "    \n",
    "top_cats_2016"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculate percentages:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46.0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# politics\n",
    "top_cats_2016['count']['Politics'] / len(top_2016) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "32.0"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# crime\n",
    "top_cats_2016['count']['Crime'] / len(top_2016) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6.0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# world\n",
    "top_cats_2016['count']['World'] / len(top_2016) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# How do the top debunking stories compare to the top fake news stories?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get top fifty debunking stories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "debunk_2017 = pd.read_csv(\"../data/fact_check.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'url', 'Politifact', 'Politifact FB', 'Snopes', 'Snopes FB',\n",
       "       'Factcheck', 'Factcheck FB', 'ABC', 'ABC FB'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debunk_2017.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "How many of the top 50 shares could we locate fact checking articles for?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_or_more_checked = debunk_2017[\n",
    "    ~pd.isna(debunk_2017['Politifact']) | \n",
    "    ~pd.isna(debunk_2017['Snopes']) | \n",
    "    ~pd.isna(debunk_2017['ABC']) | \n",
    "    ~pd.isna(debunk_2017['Factcheck'] )\n",
    "]\n",
    "\n",
    "len(one_or_more_checked)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the total engagement for all fact checking articles in the top 50?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'url', 'Politifact', 'Politifact FB', 'Snopes', 'Snopes FB',\n",
       "       'Factcheck', 'Factcheck FB', 'ABC', 'ABC FB'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debunk_2017.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "127543.0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debunk_shares = \\\n",
    "    debunk_2017['Politifact FB'].sum() + \\\n",
    "    debunk_2017['Snopes FB'].sum() + \\\n",
    "    debunk_2017['Factcheck FB'].sum() + \\\n",
    "    debunk_2017['ABC FB'].sum()\n",
    "\n",
    "debunk_shares"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What is the total engagement for all in the top 50?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'url', 'domain', 'fb_engagements', 'published_date',\n",
       "       'category'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2017.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "23498652"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2017_shares = top_2017['fb_engagements'].sum()\n",
    "top_2017_shares"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What's the percent difference?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.54276730426919806"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "debunk_shares/top_2017_shares * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Which months saw more top posts in 2017?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the date column to `datetime` and re-index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_2017['published_date'] = pd.to_datetime( top_2017['published_date'] )\n",
    "# re-index by date\n",
    "top_2017.index = top_2017['published_date']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Group by month and convert the integer month to a named month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>published_date</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>January</th>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>February</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>March</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>April</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>May</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>June</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>July</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>August</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>September</th>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>October</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>November</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>December</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                title\n",
       "published_date       \n",
       "January             8\n",
       "February            3\n",
       "March               3\n",
       "April               4\n",
       "May                 9\n",
       "June                3\n",
       "July                5\n",
       "August              2\n",
       "September           6\n",
       "October             2\n",
       "November            4\n",
       "December            1"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2017_bymonth = top_2017.groupby(pd.Grouper(freq='M')) \\\n",
    "     .count()\n",
    "    \n",
    "top_2017_bymonth.index = top_2017_bymonth.index.map( lambda x: calendar.month_name[x.month] )\n",
    "\n",
    "top_2017_bymonth['title'].to_frame()\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** First six months vs. 2017 vs last six months **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first half of 2017\n",
    "top_2017_bymonth.iloc[0:6, 0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# second half of 2017\n",
    "top_2017_bymonth.iloc[6:, 0].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot the distribution of posts by month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x10e4cbc88>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAETCAYAAADqC8daAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFA9JREFUeJzt3X2QZXV95/H3B0aUESFxw5KBYVCsZY1oRER0JcpN0BF1\n1qwxWxtIVaJurNqVXXB1rbhmC3qo2qzkYRNK12y5UTYCuisGI07W1Ph0dUUjCAwMj1oGdx4YKQ0I\ni1jy9N0/zpmhbWamb9P3ND+636+qrj5z7++c7/d2T3/u6d+9fX6pKiRJbTngiW5AkvRYhrMkNchw\nlqQGGc6S1CDDWZIaZDhLUoMmCuck5yTZ2n+cPXRTkrTSzRvOSY4H/iVwEnACsCHJsUM3Jkkr2SRn\nzr8AfKOqflJVDwNfAX5t2LYkaWWbJJxvBF6R5GeTrAZeBxw9bFuStLKtmm9AVd2a5ALgc8B9wHXA\nw0M3JkkrWRZ6bY0k/wnYXlX/bc7tXqRDkhaoqrK32yd9t8bh/ed1wBuBj+2jyII/zjvvvMe1X+u1\nlks9gPXnb9rrx7GjM/Z53+P9/7Dcv57WW571Hm+t/Zl3WqP3l0meCTwIvL2q7p1wP0nS4zBROFfV\nK4duRJL0qCf8LwRHo9GyrLUS6j3z2S9Y0nrL/etpvSdvvSFqLfgFwX0eKKlpHUvtSLJnDnkhNp+7\nYd45NWmlS0It5gVBSdLSMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalB\nhrMkNchwlqQGGc6S1CDDWZIaNOkyVf8hyU1JbkhyaZKDhm5MklayecM5yTHA24AXVdUv0q2e8htD\nNyZJK9kky1TdCzwAPD3JI8Bq4I5Bu5KkFW7eM+equhv4Y2AbsBP4YVV9fujGJGklm2Ra41jg3wHH\nAEcChyQ5c+jGJGklm2Ra4yTgyqq6CyDJ5cDLgY/NHTgzM7NnezQa7Vn08Mi169i1c/uCm1tz1NHc\nsWPbgveTpBaNx2PG4/FEY+dd4DXJC4FLgJcAPwEuAq6uqv86Z9w+F3h1kdAnL7930nAWtcBrVV0P\nfBS4BrgeCPChqXYoSfopk0xrUFV/CPzhwL1Iknr+haAkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lq\nkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNMpwlqUGTLPB6XJLrklzb\nf74nydlL0ZwkrVTzroRSVd8CXgSQ5ABgB/CpgfuSpBVtodMarwK+U1ULX0pbkjSxhYbzvwA+PkQj\nkqRHTRzOSZ4CvAG4bLh2JEkw4erbvdcC11TV9/c1YGZmZs/2aDRiNBo97sYkabkZj8eMx+OJxi4k\nnM9gnimN2eEsSfppc09aN27cuM+xE01rJFlN92Lg5YvsTZI0gYnOnKvqfuDwgXuRJPX8C0FJapDh\nLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S\n1CDDWZIaZDhLUoMmXQnlsCSXJbklyU1JXjp0Y5K0kk26huCFwP+uqn+eZBWwesCeJGnFmzeckxwK\nvKKq3gxQVQ8B9w7clyStaJNMazwb+EGSi5Jcm+RDSQ4eujFJWskmmdZYBZwInFVV30zyp8B7gPPm\nDpyZmdmzPXcJcElL78i169i1c/uC91tz1NHcsWPbAB2tbOPxmPF4PNHYVNX+ByRHAF+vqmP7f/8S\n8LtV9U/njKt9HSsJ68/fNFFDs20+dwPz9adh+b17cvP717YkVFX2dt+80xpVdSewPclx/U2nATdP\nsT9J0hyTvlvjbODSJE8B/g54y3AtSZImCuequh54ycC9SJJ6/oWgJDXIcJakBhnOktQgw1mSGmQ4\nS1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDZroes5Jvgvc\nAzwCPFhVJw/ZlCStdJOuhPIIMKqqu4dsRpLUmXRaIwsYK0lapEkDt4DPJbk6yduGbEiSNPm0xilV\ntSvJ4XQhfUtVfXXIxiRpJZt0gddd/efvJ/kUcDLwmHCemZnZsz0ajRiNRlNpUhrKkWvXsWvn9gXv\nt+aoo7ljx7YBOtJyNh6PGY/HE42dN5yTrAYOqKr7kjwdWA9s3NvY2eEsPRns2rmd9edvWvB+m8/d\nMEA3Wu7mnrRu3LjXKAUmO3M+AvhUkurHX1pVmxfZoyRpP+YN56q6HThhCXqRJPV8e5wkNchwlqQG\nGc6S1CDDWZIaZDhLUoMMZ0lqkOEsSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDh\nLEkNMpwlqUETh3OSA5Jcm+SKIRuSJC3szPkc4OahGpEkPWqicE6yFngd8OfDtiNJgsnPnP8EeDdQ\nA/YiSerNG85JXg/cWVVbgPQfkqQBzbv6NnAK8IYkrwMOBp6R5KNV9VtzB87MzOzZHo1GjEajKbWp\n3Y5cu45dO7cveL81Rx3NHTu2DdCR9MR5PD8PT+TPwng8ZjweTzR23nCuqvcC7wVIcirwrr0FM/x0\nOGsYu3ZuZ/35mxa83+ZzNwzQjfTEejw/D0/kz8Lck9aNGzfuc6zvc5akBk0yrbFHVX0Z+PJAvUiS\nep45S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalB\nhrMkNchwlqQGGc6S1CDDWZIaNO/F9pM8FfgKcFD/8el+6SpJ0kAmWUPwJ0l+uaruT3IgcGWSU6rq\nyiXoT5JWpImmNarq/n7zqf0+dw/WkSRpsnBOckCS64DvAeOqunnYtiRpZZv0zPmRqnoRsBZ4ZZJT\nh21Lkla2ha6+fW+SvwZOYi+rcM/MzOzZHo1GjEajRba3cEeuXceundsXvN+ao47mjh3bmq+33Pn1\n1HI2Ho8Zj8cTjZ3k3Ro/BzxYVfckORh4NbBxb2Nnh/MTZdfO7aw/f9OC99t87oYnRb3lzq+nlrO5\nJ60bN+41SoHJzpzXAH+RJHTTIBdX1RcW2aMkaT8meSvdVuDEJehFktTzLwQlqUGGsyQ1yHCWpAYZ\nzpLUIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEs\nSQ2aN5yTrE3yxSQ3Jdma5OylaEySVrJJlql6CHhnVW1JcghwTZLNVXXrwL1J0oo175lzVX2vqrb0\n2/cBtwBHDd2YJK1kC5pzTvIs4ATgG0M0I0nqTBzO/ZTGJ4Fz+jNoSdJAJplzJskqumC+uKo+va9x\nMzMze7ZHoxGj0WiR7UnLy5Fr17Fr5/YF77fmqKO5Y8e2ATqaruX++BZrPB4zHo8nGjtROAMfAW6u\nqgv3N2h2OEt6rF07t7P+/E0L3m/zuRsG6Gb6lvvjW6y5J60bN27c59hJ3kp3CvCbwK8kuS7JtUlO\nn0KfkqR9mPfMuaquBA5cgl4kST3/QlCSGmQ4S1KDDGdJapDhLEkNMpwlqUGGsyQ1yHCWpAYZzpLU\nIMNZkhpkOEtSgwxnSWqQ4SxJDTKcJalBhrMkNchwlqQGGc6S1KBJVkL5cJI7k9ywFA1JkiY7c74I\neM3QjUiSHjVvOFfVV4G7l6AXSVLPOWdJatC8C7wuxMzMzJ7tuUuAS9KT2ZFr17Fr5/YF77fmqKO5\nY8c2AMbjMePxeKL9BgtnSVpOdu3czvrzNy14v83nbtizPfekdePGjfvcb9JpjfQfkqQlMMlb6T4G\nfA04Lsm2JG8Zvi1JWtnmndaoqjOXohFJ0qN8t4YkNchwlqQGGc6S1CDDWZIaZDhLUoMMZ0lqkOEs\nSQ0ynCWpQYazJDXIcJakBhnOktQgw1mSGmQ4S1KDDGdJapDhLEkNmiick5ye5NYk30ryu0M3JUkr\n3SQroRwAfAB4DXA8cEaS506rgbtuv2Fah2qqlvWsZ72VU2+IWpOcOZ8MfLuq/m9VPQj8T+BXp9XA\nXbdvndahmqplPetZb+XUG6LWJOF8FDB7PfAd/W2SpIH4gqAkNShVtf8BycuAmao6vf/3e4Cqqgvm\njNv/gSRJj1FV2dvtk4TzgcBtwGnALuAq4IyqumXaTUqSOqvmG1BVDyf5N8BmummQDxvMkjSsec+c\nJUlLzxcEJalBhrMkNWhZh3OS5yY5Lckhc24/faB6pyR5Xr99apJ3JTltiFr7qP/RJaz1S0nemWT9\nQMd/aZJD++2Dk2xM8pkkFyQ5bIB6Zyc5etrH3Uetg5L8VpJX9f8+M8kHkpyV5CkD1Tw2yb9PcmGS\n/5LkX+3++qpNzcw5J3lLVV00xeOdDZwF3AKcAJxTVZ/u77u2qk6cVq3+mL8P/ArdE94YeCXw18Cr\ngSuq6o+mXO+KuTcBvwx8EaCq3jDleldV1cn99tvovrafAtYDn6mq90253k3AC6vqoSQfAu4HPkn3\nrqEXVtWvTbnePcCPgO8AHwcuq6rvT7PGrFqX0r0Yvxr4IXAIcDndY0tV/faU650NbAC+ArwOuK6v\n+0bg7VU1nmY9TUlVNfEBbJvy8bYCh/TbzwK+SRfQANcN0P9NwIF0P3D3Aof2tx8MXD9AvWuBS4AR\ncGr/eVe/feoA9a6btX01cHi//XRg6wD1bpn9WOfct2WIx0f3xLoe+DDwfeBvgN8GnjHlWjf0n1cB\ndwIH9v/O7vumXG/rrBqrgXG/vW6In4X+2IcB7wNuBe4C/p7uROl9wM8MUXM/vXx2gGMeCvxn4GLg\nzDn3fXAaNeZ9K900JdnX1UECHDHlcgdU1X0AVfXdJCPgk0mO6etN2wNV9TBwf5LvVNW9fe0fJ3lk\ngHonAecAvwe8u6q2JPlxVX15gFoAByT5WboAO7D6s8qq+lGShwaod+Os36auT3JSVX0zyXHAgwPU\nq6p6hO4to5v76YXXAmcAfwQcPsVaByY5iO6JbTVdkN0FPJXuCX4Iq4CH+xqHAFTVtqGmUYBP0P0W\nN6qq7wEk+Xm6J7tP0D0JTk2Sff0mHLrfnKftIuDbwF8Cb03yJrqQ/gnwsmkUWNJwpgvg1wB3z7k9\nwNemXOvOJCdU1RaAqrovyQbgI8ALplwL4IEkq6vqfuDFu2/s50enPnfUB8mfJLms/3wnw34/DwOu\nofteVZI1VbWrn88f4snud4ALk/xH4AfA15Nsp7vOy+8MUO+nHkN1F/m6Argiyeop17qE7ozyAeCd\nwFeTXEn3Q/0/plwL4M+Bq5N8A3gFcAFAksPpnhSG8Kya81fEfUhfkOStA9S7Gvgye/+/+DMD1HtO\nVb2p3/6rJL8HfDHJ1KYTl3TOOcmHgYuq6qt7ue9jVXXmFGutBR7a/aw9575TqurKadXqj/nU/llz\n7u0/B6ypqkEvkZXk9cApVfXeIevspe5q4Iiqun2g4x8KPJvuiWdHVd05UJ3jqupbQxx7H/WOAe6t\nqruTHEv3m9BtVXX9QPWOB34BuLGqbh2ixpx6m4HPA3+x+3uW5AjgzcCrq+pVU653I/DGqvr2Xu7b\nXlVTfbE3yS3A8f1J0u7b3gy8m2469ZhF11jKcJa0MvRTYO+hu7zwP+xvvpPut5H3VdXc354XW+/X\n6V77uG0v9/2zqvqrKdf7A2BzVX1+zu2nA++vqn+06BqGs6SlNO13Zi3XeoazpCWVZFtVrbPe/i31\nC4KSVoAlfmfWsqxnOEsawlK+M2tZ1jOcJQ1hE927FrbMvSPJ2Hrzc85Zkhq0rC98JElPVoazJDXI\ncJakBhnOktQgw1lPqCTnJXnnXm4/JsnWfvvFSf50P8c4NclnFtHDnloTjv/Sfq6CtnvMOUme9nh7\nkgxntawAquqaqnrHJGMXW2uK3kF3OVDpcTGcNVX9WegtSS5JcnOST/TLTN2e5Jn9mBcn+dKs3U5I\n8rUktyV5zOVAZ58Z99vXJbk2yTVJnt4Pe0aSy/raF8/a98Qk4yRXJ/lsf2W03T1sSXId3aou+3tM\nT0vy8SQ3JbkceNqs+z6Y5KokW5Oc19/2b4EjgS8l+UJ/2/r+MX4zyf8a4DKkWmYMZw3hHwMfqKrn\n0a0K83Yee2Y6+98voFvJ5eXAuf1F2efaPf5ddEsrnUh3beIf97efAJwNPA94TpKXJ1kFvB94U1W9\nhO4C6b/fj/8IcFZVvWiCx/OvgR9V1fHAeXSX99ztvdUt3/VCYJTk+VX1fmAn3YXmT0vyD+gWRTit\nqk6iuy72uyaoqxXMvxDUELZV1d/225fSheb+fLqqHgD+PskXgZOBfV3X+Eq6xQUuBS6vqp1JAK6q\nql0ASbbQLU12D/B84HPpBh0A3NEvgHDYrGt6Xwzsb9HfVwIXAlTV1iSze/uNdGsqrgJ+nu7J4Ua6\nP+PdfeH3l/W3X9n38RTg6/N8TbTCGc5aCgU8xKO/qc19oWz2WXTYz/xvVV2QZBPwerqw273c0eyF\nDh6m+78duovLnzL7GFn86t3pj/MsujPgF1fVvUku4rGPbff4zVX1m4usqxXEaQ0NYV2Sl/bbZwL/\nB/guj04HvGnO+F9NclD/6/+pdEsOwV6WHEpybFXdVFV/0I977n76uA04PMnL+n1XJXleVd0D/DDJ\ny/tx84XmV3aPSfJ84Bf72w8F7gP+Xz+X/dpZ+9zb3w/wt8ApSZ7TH2N1kkVfjF3Lm+GsIdwGnJXk\nZrr12/4MOJ9uTcCr6M6iZ7sBGNNdzev8WUuL7e0M+h39i2/X063B99m9jNn9Lo8HgV+nW7duC90K\n2/+kH/NW4INJrp3g8fwZcEiSm4AZupXcqaobgC10q0pfAsxefu2/A3+T5AtV9QPgLcDH+76/Rjcv\nL+2TFz7SVKVbG29TVQ2xiK60YnjmrCH4jC8tkmfOUq9/cfECHn1yCfB3VTV3jlwanOEsSQ1yWkOS\nGmQ4S1KDDGdJapDhLEkNMpwlqUH/Hwzd27vXV+3pAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e4a07b8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "top_2017.groupby(\n",
    "    top_2017.index.month\n",
    ").count()['title'].plot(kind=\"bar\", color=\"steelblue\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Which months saw more top posts in 2016?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Add an index where the month string is converted to a datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fix_date(x):\n",
    "    m = x['month']\n",
    "    d = datetime.strptime(m, \"%B\").replace(year=2016)\n",
    "    \n",
    "    return d\n",
    "\n",
    "top_2016.index = top_2016.apply( fix_date , axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Group by month and count the titles. Order is preserved because of datetime. Now convert back to month name for readability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>January</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>February</th>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>March</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>April</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>May</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>June</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>July</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>August</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>September</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>October</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>November</th>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           title\n",
       "January        2\n",
       "February       4\n",
       "March          7\n",
       "April          5\n",
       "May            3\n",
       "June           0\n",
       "July           9\n",
       "August         1\n",
       "September      3\n",
       "October       11\n",
       "November       5"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_2016_bymonth = top_2016.groupby(pd.Grouper(freq=\"M\")) \\\n",
    "    .count()['title'] \\\n",
    "    .to_frame()\n",
    "    \n",
    "top_2016_bymonth.index = top_2016_bymonth.index.map( \n",
    "    lambda x: calendar.month_name[x.month] \n",
    ")\n",
    "\n",
    "top_2016_bymonth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "top_2016_bymonth.to_clipboard()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare first six months to last six months:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first half of 2016\n",
    "top_2016_bymonth.iloc[0:6, 0].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# last half of 2016\n",
    "top_2016_bymonth.iloc[6:, 0].sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot by month"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x10e573048>"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW0AAAEDCAYAAAD+/1UIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADp1JREFUeJzt3X2MbHV9x/H3B65arwiaVJFlXah/UFtjQ621ptQyFQWK\nxIekTSpNVExMmtpAqjWiJrDXPxqottbU9A8jUp9oo9RWpLW9JWZowNQnuIAIaKztveD11haU+BAR\n+faPGcl1dvfuzZyzZ+e3vF/J5J45c/b8Pnt35zNnz5w5J1WFJKkNx2x3AEnS0bO0JakhlrYkNcTS\nlqSGWNqS1BBLW5IasmlpJ7kyyaEktx0278+S3JlkX5K/T3L81saUJMHRbWlfBZwzM28v8KyqOh34\nKvCWvoNJktbatLSr6kbg/pl511fVw9O7/wEsb0E2SdKMPvZpvxb4VA/rkSRtolNpJ3kb8KOqurqn\nPJKkI9g17xcmeQ1wHvDCTZbz5CaSNIeqyuy8o93SzvQ2uZOcC7wJeGlV/fAoBu50u+yyyzqvYydk\nWJQci5BhUXIsQoZFybEIGRYlRx8ZNnI0h/xdDXwGOC3J/iQXAn8FHAf8W5Kbk/z1prUvSeps090j\nVXXBOrOv2oIskqRNNPGJyNFotN0RFiIDLEaORcgAi5FjETLAYuRYhAywGDm2MkOOtO+klwGS2uox\nJGmnSUJ1eCNSkrQALG1JaoilLUkNsbQlqSGWtiQ1xNKWpIZY2pLUEEtbkhpiaUtSQyxtSWqIpS1J\nDbG0JakhlrakZiwtr5Ck021peWW7v41OPMufpGYk4ey3X9dpHXsvPf+IV4ZZFJ7lT5J2AEtbkhpi\naUtSQyxtSWqIpS1JDbG0JakhlrYkNcTSlqSGWNqS1BBLW5IaYmlLUkMsbUlqyKalneTKJIeS3HbY\nvCcn2Zvk7iT/muSErY0pSYKj29K+CjhnZt4lwPVV9fPAp4G39B1MkrTWpqVdVTcC98/Mfhnwgen0\nB4CX95xLkrSOefdpP7WqDgFU1TeBp/YXSZK0kb7eiFz8M4pL0g6wa86vO5TkxKo6lORpwP8caeHV\n1dVHpkejEaPRaM5hJWlnGo/HjMfjTZc7qsuNJTkV+GRVPXt6/wrgvqq6IsmbgSdX1SUbfK2XG5PU\nCy83dnSH/F0NfAY4Lcn+JBcClwMvTnI3cNb0viRpi226e6SqLtjgoRf1nEWStAk/ESlJDbG0Jakh\nlrYkNcTSlqSGWNqS1BBLW5IaYmlLUkMsbUlqiKUtSQ2xtCWpIZa2JDXE0pakhljaktQQS1uSGmJp\nS1JDLG3pCJaWV0gy921peWW7vwXtMPNeI1J6VDh474FOl7fae+n5PaaR3NKWpKZY2pLUEEtbkhpi\naUtSQyxtSWqIpS1JDbG0JakhlrYkNcTSlqSGWNqS1BBLW5IaYmlLUkM6lXaStyS5I8ltST6S5LF9\nBZMkrTV3aSc5BXgd8MtV9UtMzhj4e30FkySt1eXUrA8ADwJPSPIwsBv4Ri+pJEnrmntLu6ruB/4c\n2A/cC3y7qq7vK5gkaa25t7STPAP4Y+AU4DvANUkuqKqrZ5ddXV19ZHo0GjEajeYdVpJ2pPF4zHg8\n3nS5LrtHngvcVFX3AST5OPDrwBFLW5K01uwG7Z49e9ZdrsvRI3cDz0/yM0kCnAXc2WF9kqRNdNmn\nfSvwQeCLwK1AgPf2lEuStI5OF/atqncA7+gpiyRpE34iUpIaYmlLUkMsbUlqiKUtSQ2xtCWpIZa2\nJDXE0pakhljaktQQS1uSGmJpS1JDLG1JaoilLUkNsbQlqSGWtiQ1xNJeIEvLKyTpdFtaXtnub0PS\nFup0Pm316+C9Bzj77dd1WsfeS8/vKY2kReSWtiQ1xNKWpIZY2pLUEEtbkhpiaUtSQyxtSWqIpS1J\nDbG0JakhlrYkNcTSlqSGWNqS1BBLW5Ia0qm0k5yQ5GNJ7kxyR5Jf6yuYJGmtrmf5ezfwz1X1u0l2\nAbt7yCRJ2sDcpZ3keOAFVfUagKp6CHigp1ySpHV02T3yc8D/Jrkqyc1J3pvk8X0FkySt1WX3yC7g\nOcDrq+oLSf4SuAS4bHbB1dXVR6ZHoxGj0ajDsJK084zHY8bj8abLdSnte4ADVfWF6f1rgDevt+Dh\npS1JWmt2g3bPnj3rLjf37pGqOgQcSHLadNZZwJfnXZ8kaXNdjx65CPhIkscA/wlc2D2SJGkjnUq7\nqm4FfrWnLJKkTfiJSElqiKUtSQ2xtCWpIZa2JDXE0pakhljaktQQS1uSGmJpS1JDLG1JaoilLUkN\nsbQlqSGWtiQ1xNKWpIZY2pLUEEtbaywtr5Ck021peWW7vw1py3R9jnR5fnS9CIJ2oIP3HuDst1/X\naR17Lz2/pzTS4un6HOny/HBLW5IaYmlLUkMsbUlqiKUtSQ2xtCWpIZa2JDXE0pakhljaktQQS1uS\nGmJpS1JDLG1JaoilLUkN6VzaSY5JcnOSa/sIJEnaWB9b2hcDX+5hPZKkTXQq7STLwHnA+/qJI0k6\nkq5b2u8C3gRUD1kkSZuYu7STvAQ4VFX7gExvkqQt1OXKNWcAL01yHvB44IlJPlhVr5pdcHV19ZHp\n0WjEaDTqMOzWWFpe4eC9Bzqt46STn8437tnfUyJJjybj8ZjxeLzpcnOXdlW9FXgrQJIzgTeuV9jw\n06W9qLzElqTtNLtBu2fPnnWX8zhtSWpILxf2raobgBv6WJckaWNuaUtSQyxtSWqIpS1JDbG0Jakh\nlrYkNcTSlqSGWNqS1BBLW5IaYmlLUkMsbUlqiKUtSQ2xtCWpIZa2JDXE0pakhljaWkhLyysk6XRb\nWl7Z7m9jR+n6M/Hn0Y9ezqct9c0rCS2erj8Tfx79cEtbkhpiaUtSQyxtSWqIpS1JDbG0JakhlrYk\nNcTSlqSGWNqS1BBLW5IaYmlLUkMsbUlqiKUtSQ2Zu7STLCf5dJI7ktye5KI+g0mS1upylr+HgDdU\n1b4kxwFfTLK3qu7qKZskacbcW9pV9c2q2jed/i5wJ3ByX8EkSWv1sk87yanA6cBn+1ifJGl9nUt7\numvkGuDi6Ra3JGmLdLpyTZJdTAr7Q1X1iY2WW11dfWR6NBoxGo0eub+0vMLBew90icFJJz+db9yz\nv9M6JGk7jcdjxuPxpst1vdzY+4EvV9W7j7TQ4aU9y8tKSdLaDdo9e/asu1yXQ/7OAH4feGGSW5Lc\nnOTcedcnSdrc3FvaVXUTcGyPWSRJm/ATkZLUEEtbkhpiaUtSQyxtSWqIpS1JDbG0JakhlrYkNcTS\nlqSGWNqS1BBLW5IaYmlLUkMsbUlqiKUtSQ2xtCWpIZa2tOCWlldI0um2tLyy3d+GetL1yjWStphX\nd9Lh3NKWpIZY2pLUEEtbkhpiaUtSQyxtSWqIpS1JDbG0JakhlrYkNcTSlqSGWNqS1BBLW5IaYmlL\nUkM6lXaSc5PcleQrSd7cVyhJ0vrmLu0kxwDvAc4BngW8Mskz+wp2uPu+fttWrLa5DLAYORYhAyxG\njkXIAIuRYxEywGLk2MoMXba0nwd8tar+u6p+BPwd8LJ+Yv20+75++1astrkMsBg5FiEDLEaORcgA\ni5FjETLAYuTYygxdSvtk4MBh9++ZzpMkbRHfiJSkhqSq5vvC5PnAalWdO71/CVBVdcXMcvMNIEmP\nclWV2XldSvtY4G7gLOAg8DnglVV1Z5eQkqSNzX2NyKr6cZI/AvYy2c1ypYUtSVtr7i1tSdLwfCNS\nkhpiaUtSQyztDSR5ZpKzkhw3M//cATOckeQXp9NnJnljkrOGGv8IuT64zeP/RpI3JDl74HEvSvL0\nIcdcJ8Njk7wqyYum9y9I8p4kr0/ymIGzPCPJnyR5d5K/SPIHSY4fMsOjUVP7tJNcWFVXDTDORcDr\ngTuB04GLq+oT08durqrnDJDhT4EXMnlhHQO/CfwT8GLg2qp651ZnmOa4dnYW8FvApwGq6qUDZPhc\nVT1vOv06Jj+bfwDOBj5ZVZdvdYbp2N8Bvgd8Dfhb4GNV9a0hxj4sw0eYHECwG/g2cBzwcSZHcaWq\nXj1QjouA84F/B84DbpnmeQXwh1U1HiLHo1JVNXMD9g80zu3AcdPpU4EvMClugFsGynAHcCyTJ+cD\nwPHT+Y8Hbh3w//xm4MPACDhz+u/B6fSZA2W45bDpzwNPmU4/Abh9wP+LW5i8iJ4NXAl8C/gX4NXA\nEwfKcNv0313AIeDY6f385LGBctx+2Ni7gfF0emWo58h0vBOAy4G7gPuA/2OysXU58KShchwh36f6\nXufch/xtlSQbnWklwIkDxTimqr4LUFX/lWQEXJPklGmOITxYVT8Gvp/ka1X1wDTPD5I8PFAGgOcC\nFwNvA95UVfuS/KCqbhgwwzFJnsykMI+t6dZtVX0vyUMD5qiqepjJYa57p7sjfht4JfBO4CkDZDg2\nyWOZvGDtZlJa9wGPY/IiP6RdwI+nYx8HUFX7B95N81Emf/WNquqbAEmexuSF9KNMXmC3VJKN/vIO\nk7/Ue7Vwpc2kmM8B7p+ZH+AzA2U4lOT0qtoHUFXfTXI+8H7g2QNleDDJ7qr6PvArP5mZ5ARgsH1a\n05J6V5KPTf89xPC/NycAX2TyO1BJTqqqg9P3G4Z6EWV2rJqcKO1a4NokuwfK8GEmW5UPAm8Abkxy\nE/B84G8GygDwPuDzST4LvAC4AiDJU5i8iAzl1Jr5FPa0vK9I8tqBMnweuIH1fxef1PdgC7dPO8mV\nwFVVdeM6j11dVRcMkGEZeOgnr9wzj51RVTcNkOFxVfXDdeb/LHBSVW3LqcySvAQ4o6reuh3jz2TZ\nDZxYVV8faLzTquorQ4y1SY5TgAeq6v4kz2Dy19DdVXXrwDmeBfwC8KWqumvIsQ/LsBe4HvhAVR2a\nzjsReA3w4qp60QAZvgS8oqq+us5jB6qq1zevF660JeloTXebXcLktNBPnc4+xOQvoMuravYv9q3I\n8DtM3lu5e53HXl5V/9jreJa2pJ1oqKPNhs5gaUvakZLsr6qVnZZhEd+IlKSjsghHmw2dwdKW1LJF\nONps0AyWtqSWXcfkg3D7Zh9IMt6JGdynLUkN8YRRktQQS1uSGmJpS1JDLG1JaoilLUkN+X+JggaC\nz2E9RgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10e5d3b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "top_2016.groupby(\n",
    "    top_2016.index.month\n",
    ") \\\n",
    ".count()['title'] \\\n",
    ".plot(kind=\"bar\", color=\"steelblue\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "---\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}