Py - SpotiPy and Genius 17K Songs and Lyrics Retrieval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e18d4190",
   "metadata": {},
   "source": [
    "# Retrieving song info + lyrics\n",
    "## A code that builds a 16K songs dataset via SpotiPy and LyricsGenius\n",
    "### The second part of the Lyrics Entropy project is available [here](https://carlosfg97.github.io/MusicLyricEntropy/Music-Lyrics-Entropy.html)\n",
    "### and the main repo is [here](https://github.com/carlosfg97/MusicLyricEntropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9aee5b51",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install spotipy\n",
    "# https://spotipy.readthedocs.io/en/2.19.0/\n",
    "# https://medium.com/@maxtingle/getting-started-with-spotifys-api-spotipy-197c3dc6353b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d3ff48e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spotipy\n",
    "import pandas as pd\n",
    "from spotipy.oauth2 import SpotifyClientCredentials\n",
    "import pickle\n",
    "cid =  # add Spotify API client ID\n",
    "secret =  # ADD secret token \n",
    "client_credentials_manager = SpotifyClientCredentials(client_id=cid, client_secret=secret)\n",
    "sp = spotipy.Spotify(client_credentials_manager = client_credentials_manager)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "11a6ccb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#taken from https://github.com/MaxHilsdorf/introduction_to_spotipy/blob/master/introduction_to_spotipy.ipynb\n",
    "def get_playlist_tracks(username,playlist_id):\n",
    "    results = sp.user_playlist_tracks(username,playlist_id)\n",
    "    tracks = results['items']\n",
    "    while results['next']:\n",
    "        results = sp.next(results)\n",
    "        tracks.extend(results['items'])\n",
    "    return tracks\n",
    "\n",
    "def get_playlist_tracks(playlist_id):\n",
    "    results = sp.playlist_tracks(playlist_id)\n",
    "    tracks = results['items']\n",
    "    while results['next']:\n",
    "        results = sp.next(results)\n",
    "        tracks.extend(results['items'])\n",
    "    return tracks"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e6c586a",
   "metadata": {},
   "source": [
    "### TRACK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e4036dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "playlist_ids = [ '532F1h299qKD894BlPfJJF', '7bVwBxsWk84iKeMPXa8oRs', '6dSeKJvN1MhLP1IEwBWFXg',\n",
    "                '0f2CvetujnCTOxB6KCXhII', '37i9dQZF1DXcBWIGoYBM5M', '37i9dQZF1DX4bSrsRWE9cd',\n",
    "                '37i9dQZF1DWU3bkMPOyjie', '37i9dQZF1DWY4xHQp97fN6', '37i9dQZF1DX4WYpdgoIcn6', \n",
    "                '37i9dQZF1DX5NsFgylu4qQ', '37i9dQZF1DX82pCGH5USnM', '37i9dQZF1DWYs83FtTMQFw',\n",
    "                '37i9dQZF1DXcGnc6d1f20P','6dSeKJvN1MhLP1IEwBWFXg', '532F1h299qKD894BlPfJJF',\n",
    "                '3UVNqcek0Hg8ne1ijJn9kc','6ZlFKcTzJVslgjudScTX4G','3NgkCmLvO7UrtMsVBEeFVU',\n",
    "                '37i9dQZF1DX4UtSsGT1Sbe','37i9dQZF1DXbTxeAdrVG2l','3NVUQ2r0Ty7eWH2AwItUE4',\n",
    "                '72LA3OR3WCoXu6ZC7opyz9','5UtamknFCkYdgfwYrCbquZ','23zeB07Gmr1X5xJLV69GI0',]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2f148e1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "West's Bests 63 songs\n",
      "Olds 314 songs\n",
      "🌀FLOW🌀 160 songs\n",
      "Today's Top Hits 50 songs\n",
      "Bliss 72 songs\n",
      "Summer Rock Vibes 73 songs\n",
      "Get Turnt 100 songs\n",
      "Chill Hits 130 songs\n",
      "Waves 78 songs\n",
      "Lounge - Soft House 202 songs\n",
      "Hot Rhythmic 90 songs\n",
      "This Is J. Cole 48 songs\n",
      "Olds 314 songs\n",
      "Longest Playlist 50/60/70/80/90/00/10/20 5727 songs\n",
      "Longest playlist everrrrrrrrrrrrrrr 1983 songs\n",
      "Longest Playlist 3596 songs\n",
      "70s Hits 305 songs\n",
      "All Out 80s 150 songs\n",
      "All Out 90s 150 songs\n",
      "LONG ASS INDIE ETC PLAYLIST 433 songs\n",
      "the longest rock playlist on spotify 9421 songs\n",
      "a totally work appropriate upbeat indie playlist that's also over SEVENTEEN hours long 276 songs\n",
      "excessively long chill songs playlist 2691 songs\n"
     ]
    }
   ],
   "source": [
    "track_lists = []\n",
    "for i in playlist_ids:\n",
    "    track_list = get_playlist_tracks(i)\n",
    "    print(sp.playlist(i)['name'], str(len(track_list)) + ' songs')\n",
    "    #print(len(track_list))\n",
    "    track_lists.extend(track_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d4de786b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tracks_ids = []\n",
    "for i in track_lists:\n",
    "    try:\n",
    "        tracks_ids.append(i['track']['id'])\n",
    "    except:\n",
    "        continue   \n",
    "\n",
    "tracks_ids = list(set(tracks_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "28c0a8e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24036"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tracks_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7b295acb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_complete_track_info(track_id):\n",
    "    \n",
    "    # basic info\n",
    "    r = sp.track(track_id)\n",
    "    album_id = r['album']['id']\n",
    "    artist_id = [i['id'] for i in r['artists']]\n",
    "    duration_ms = r['duration_ms']\n",
    "    explicit = r['explicit']\n",
    "    url = r['href'] \n",
    "    song_id = r['id']\n",
    "    song_name = r['name']\n",
    "    song_popularity = r['popularity']\n",
    "    song_type = r['type']\n",
    "    song_release_date = r['album']['release_date']\n",
    "    \n",
    "    \n",
    "    # track audio analysis attributes\n",
    "    af = sp.audio_features([track_id])[0]\n",
    "    track_danceability = af['danceability']\n",
    "    track_energy = af['energy']\n",
    "    track_key = af['key']\n",
    "    track_loudness = af['loudness']\n",
    "    track_mode = af['mode']\n",
    "    track_speechiness = af['speechiness']\n",
    "    track_acousticeess = af['acousticness']\n",
    "    track_instrumentalness = af['instrumentalness']\n",
    "    track_liveness = af['liveness']\n",
    "    track_valence = af['valence']\n",
    "    track_tempo = af['tempo']\n",
    "    \n",
    "    output = {\n",
    "        'track_id' : track_id,\n",
    "        'album_id' : album_id,\n",
    "        'artist_id' : artist_id,\n",
    "        'duration_ms' : duration_ms,\n",
    "        'flg_explicit' : explicit,\n",
    "        'url' : url,\n",
    "        'track_name' : song_name,\n",
    "        'track_popularity' : song_popularity,\n",
    "        'track_type' : song_type,\n",
    "        'track_danceability' : track_danceability,\n",
    "        'track_energy' : track_energy,\n",
    "        'track_key' : track_key,\n",
    "        'track_loudness' : track_loudness,\n",
    "        'track_mode' : track_mode,\n",
    "        'track_speechiness' : track_speechiness,\n",
    "        'track_acousticeess' : track_acousticeess,\n",
    "        'track_instrumentalness' : track_instrumentalness,\n",
    "        'track_liveness' : track_liveness,\n",
    "        'track_valence' : track_valence,\n",
    "        'track_tempo' : track_tempo   , \n",
    "        'track_release_date' : song_release_date\n",
    "    }\n",
    "    \n",
    "    return output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "1d8612d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "partition_size = int(len(tracks_ids)/4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1e555a30",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24036"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tracks_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b9d6b091",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6009 12018 18027\n"
     ]
    }
   ],
   "source": [
    "p1 = 0 + partition_size*1\n",
    "p2 = 0 + partition_size*2\n",
    "p3 = 0 + partition_size*3\n",
    "#p4 = print(0 + partition_size*4)\n",
    "print(p1,p2,p3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "298314c7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "4000\n",
      "5000\n",
      "6000\n"
     ]
    }
   ],
   "source": [
    "\n",
    "track_info = []\n",
    "f = 0 \n",
    "for i in tracks_ids[:p1]:\n",
    "    try:\n",
    "        track_info.append(get_complete_track_info(i))\n",
    "    except:\n",
    "        continue   \n",
    "        \n",
    "    f += 1\n",
    "    if f in [1000,2000,4000,5000,6000]:\n",
    "        print(f)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9c7bfc43",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tracks1 = pd.DataFrame(track_info)\n",
    "df_tracks1.to_csv('df_tracks1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "af7929f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "3000\n",
      "4000\n"
     ]
    }
   ],
   "source": [
    "track_info = []\n",
    "f=0\n",
    "\n",
    "for i in tracks_ids[p1:p2]:\n",
    "    try:\n",
    "        track_info.append(get_complete_track_info(i))\n",
    "    except:\n",
    "        continue  \n",
    "    \n",
    "    f += 1\n",
    "    if f in [1000,2000,3000,4000]:\n",
    "        print(f)\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c62dc861",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tracks2 = pd.DataFrame(track_info)\n",
    "df_tracks2.to_csv('df_tracks2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d9983798",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "4000\n",
      "5000\n"
     ]
    }
   ],
   "source": [
    "track_info = []\n",
    "f = 0\n",
    "\n",
    "for i in tracks_ids[p2:p3]:\n",
    "    try:\n",
    "        track_info.append(get_complete_track_info(i))\n",
    "    except:\n",
    "        continue   \n",
    "    f += 1\n",
    "    if f in [1000,2000,4000,5000]:\n",
    "        print(f)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "88c55d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tracks3 = pd.DataFrame(track_info)\n",
    "df_tracks3.to_csv('df_tracks3.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f2ab5ea2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "2000\n",
      "4000\n",
      "5000\n"
     ]
    }
   ],
   "source": [
    "track_info = []\n",
    "f = 0\n",
    "for i in tracks_ids[p3:]:\n",
    "    try:\n",
    "        track_info.append(get_complete_track_info(i))\n",
    "    except:\n",
    "        continue   \n",
    "    f += 1\n",
    "    if f in [1000,2000,4000,5000]:\n",
    "        print(f)\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "aec0aa92",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tracks4 = pd.DataFrame(track_info)\n",
    "df_tracks4.to_csv('df_tracks4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e5d5321b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('df_tracks1.csv')\n",
    "df2 = pd.read_csv('df_tracks2.csv')\n",
    "df3 = pd.read_csv('df_tracks3.csv')\n",
    "df4 = pd.read_csv('df_tracks4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bf212aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_alltracks.artist_id = df_alltracks.artist_id.apply(lambda x: x[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbf74748",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('df_alltracks.pickle', 'wb') as handle:\n",
    "    pickle.dump(df_alltracks, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "965ba920",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('df_alltracks.pickle', 'rb') as handle:\n",
    "    df_alltracks = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a8038d1",
   "metadata": {},
   "source": [
    "### ARTIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "60b45ac4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_tracks = df_alltracks.copy()\n",
    "artist_ids  = df_tracks.artist_id.drop_duplicates()\n",
    "artist_ids = list(set(artist_ids))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "5c67c370",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_complete_artist_info(artist_id):\n",
    "    \n",
    "    # basic info\n",
    "    a = sp.artist(artist_id)\n",
    "    artist_followers =  a['followers']['total']\n",
    "    artist_genres = a['genres']\n",
    "    artist_name = a['name']\n",
    "    artist_popularity = a['popularity']\n",
    "    artist_url = a['href']\n",
    "    \n",
    "    \n",
    "    output = {\n",
    "        'artist_id' : artist_id,\n",
    "        'artist_followers' : artist_followers,\n",
    "        'artist_genres' : artist_genres,\n",
    "        'artist_name' : artist_name,\n",
    "        'artist_popularity' : artist_popularity,\n",
    "        'artist_url' : artist_url\n",
    "    }\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "874b039e",
   "metadata": {},
   "outputs": [],
   "source": [
    "artist_info = []\n",
    "for i in artist_ids:\n",
    "    artist_info.append(get_complete_artist_info(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "be8f959e",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_artist = pd.DataFrame(artist_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3ca8d149",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_artist = df_artist[['artist_id', 'artist_name',    'artist_genres'       , 'artist_followers', 'artist_popularity']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7c47fdf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_artist.to_csv('artists.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a62587c6",
   "metadata": {},
   "source": [
    "### TRACKS LYRICS (GENIUS API)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03a893dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install lyricsgenius\n",
    "# https://pypi.org/project/lyricsgenius/\n",
    "import lyricsgenius\n",
    "genius.verbose = False # Turn off status messages\n",
    "genius.remove_section_headers = True "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57fcf903",
   "metadata": {},
   "outputs": [],
   "source": [
    "token = ### insert personal token \n",
    "genius = lyricsgenius.Genius(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00f05378",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('df_alltracks.pickle', 'rb') as handle:\n",
    "    df_alltracks = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6071d0a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "artists = pd.read_csv('artists.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f77beb9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_alltracks = df_alltracks[df_alltracks.artist_id.apply(lambda x: len(x)) == 1]\n",
    "df_alltracks.artist_id = df_alltracks.artist_id.apply(lambda x: x[0])\n",
    "print(df_alltracks.shape)\n",
    "df_alltracks = df_alltracks.merge(artists ,  how='left', left_on = 'artist_id', right_on = 'artist_id')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc019bed",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyrics_ls = []\n",
    "\n",
    "f = 0\n",
    "for song, artist_ in zip(df_alltracks.track_name[:] ,  df_alltracks.artist_name[:]):\n",
    "    try:\n",
    "        lyrics_ls.append(genius.search_song(title = song, artist = artist_ ).lyrics )\n",
    "    except:\n",
    "        lyrics_ls.append('')\n",
    "        \n",
    "    f += 1    \n",
    "    if f in [333*i for i in range(1,15)]:\n",
    "        print(f)\n",
    "    if f == 4000:\n",
    "        break\n",
    "        \n",
    "df_4k = df_alltracks[:4000].copy()\n",
    "df_4k['lyrics'] = lyrics_ls\n",
    "df_4k.to_csv('tracks4k.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b6044ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyrics_ls_2 = []\n",
    "\n",
    "f = 0\n",
    "for song, artist_ in zip(df_alltracks.track_name[4000:8000] ,  df_alltracks.artist_name[4000:8000]):\n",
    "    try:\n",
    "        lyrics_ls.append(genius.search_song(title = song, artist = artist_ ).lyrics )\n",
    "    except:\n",
    "        lyrics_ls.append('')\n",
    "        \n",
    "    f += 1    \n",
    "    if f in [333*i for i in range(1,15)]:\n",
    "        print(f)\n",
    "        \n",
    "df_4k = df_alltracks[4000:8000].copy()\n",
    "df_4k['lyrics'] = lyrics_ls[4000:8000]\n",
    "\n",
    "df_4k.to_csv('tracks4_8k.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72c772fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyrics_ls_3 = []\n",
    "\n",
    "f = 0\n",
    "for song, artist_ in zip(df_alltracks.track_name[8000:12000] ,  df_alltracks.artist_name[8000:12000]):\n",
    "    try:\n",
    "        lyrics_ls_3.append(genius.search_song(title = song, artist = artist_ ).lyrics )\n",
    "    except:\n",
    "        lyrics_ls_3.append('')\n",
    "        \n",
    "    f += 1    \n",
    "    if f in [333*i for i in range(1,15)]:\n",
    "        print(f)\n",
    "        \n",
    "df_4k = df_alltracks[8000:12000].copy()\n",
    "df_4k['lyrics'] = lyrics_ls_3\n",
    "\n",
    "df_4k.to_csv('tracks8_12k.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "583cbb8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyrics_ls_4 = []\n",
    "\n",
    "f = 0\n",
    "for song, artist_ in zip(df_alltracks.track_name[12000:16000] ,  df_alltracks.artist_name[12000:16000]):\n",
    "    try:\n",
    "        lyrics_ls_4.append(genius.search_song(title = song, artist = artist_ ).lyrics )\n",
    "    except:\n",
    "        lyrics_ls_4.append('')\n",
    "        \n",
    "    f += 1    \n",
    "    if f in [333*i for i in range(1,15)]:\n",
    "        print(f)\n",
    "        \n",
    "df_4k = df_alltracks[12000:16000].copy()\n",
    "df_4k['lyrics'] = lyrics_ls_4\n",
    "\n",
    "df_4k.to_csv('tracks12_16k.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa7f2381",
   "metadata": {},
   "outputs": [],
   "source": [
    "lyrics_ls_5 = []\n",
    "\n",
    "f = 0\n",
    "for song, artist_ in zip(df_alltracks.track_name[16000:20000] ,  df_alltracks.artist_name[16000:20000]):\n",
    "    try:\n",
    "        lyrics_ls_5.append(genius.search_song(title = song, artist = artist_ ).lyrics )\n",
    "    except:\n",
    "        lyrics_ls_5.append('')\n",
    "        \n",
    "    f += 1    \n",
    "    if f in [333*i for i in range(1,15)]:\n",
    "        print(f)\n",
    "        \n",
    "df_4k = df_alltracks[16000:20000].copy()\n",
    "df_4k['lyrics'] = lyrics_ls_5\n",
    "\n",
    "df_4k.to_csv('tracks16_20k.csv', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "b6be6ccd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('tracks4k.csv')\n",
    "df2 = pd.read_csv('tracks4_8k.csv')\n",
    "df3 = pd.read_csv('tracks8_12k.csv')\n",
    "df4 = pd.read_csv('tracks12_16k.csv')\n",
    "df5 = pd.read_csv('tracks16_20k.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e7d022b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_all = pd.concat([df1,df2,df3,df4, df5])\n",
    "df_all = df_all[ ~ df_all.lyrics.isna() ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "32d3b67d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-             1796\n",
       "The           1544\n",
       "You           1189\n",
       "I              823\n",
       "the            766\n",
       "Me             754\n",
       "Love           690\n",
       "My             554\n",
       "Remastered     540\n",
       "of             495\n",
       "Remaster       465\n",
       "A              460\n",
       "Of             442\n",
       "In             438\n",
       "It             390\n",
       "On             360\n",
       "To             346\n",
       "to             336\n",
       "Your           298\n",
       "a              293\n",
       "Version        281\n",
       "Don't          266\n",
       "All            256\n",
       "in             256\n",
       "Be             242\n",
       "Is             239\n",
       "Live           220\n",
       "For            203\n",
       "Like           196\n",
       "Time           187\n",
       "One            183\n",
       "No             182\n",
       "And            178\n",
       "and            169\n",
       "I'm            165\n",
       "This           164\n",
       "Down           162\n",
       "2011           162\n",
       "Man            161\n",
       "What           158\n",
       "Up             147\n",
       "Out            146\n",
       "Way            137\n",
       "Back           137\n",
       "Get            136\n",
       "Go             135\n",
       "Single         131\n",
       "Song           130\n",
       "on             130\n",
       "Life           129\n",
       "Heart          127\n",
       "Never          127\n",
       "Do             125\n",
       "Girl           125\n",
       "We             124\n",
       "Rock           123\n",
       "for            123\n",
       "Night          122\n",
       "Little         122\n",
       "World          121\n",
       "dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = df_all.track_name.to_list()\n",
    "words = []\n",
    "for name in names:\n",
    "    words.extend(str(name).split(' '))\n",
    "    \n",
    "pd.Series(words).value_counts().head(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5105706b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16762, 26)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all = df_all[ ~ df_all.track_name.astype(str).str.contains('Remaster')]\n",
    "df_all = df_all[ ~ df_all.track_name.astype(str).str.contains('Version')].reset_index(drop=True)\n",
    "df_all = df_all[ ~ df_all.track_name.astype(str).str.contains('Live')].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7421019f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "The       1381\n",
       "You       1064\n",
       "I          737\n",
       "Me         687\n",
       "the        667\n",
       "Love       618\n",
       "My         496\n",
       "of         438\n",
       "A          412\n",
       "-          405\n",
       "Of         401\n",
       "In         394\n",
       "It         346\n",
       "On         328\n",
       "To         300\n",
       "to         295\n",
       "Your       267\n",
       "a          263\n",
       "All        234\n",
       "Don't      234\n",
       "Be         222\n",
       "Is         208\n",
       "in         205\n",
       "For        190\n",
       "Like       182\n",
       "Time       167\n",
       "No         167\n",
       "One        162\n",
       "And        154\n",
       "I'm        150\n",
       "This       147\n",
       "Down       146\n",
       "and        146\n",
       "What       139\n",
       "Man        138\n",
       "Up         137\n",
       "Out        130\n",
       "Go         124\n",
       "Heart      123\n",
       "Back       123\n",
       "Way        118\n",
       "Song       117\n",
       "Life       115\n",
       "Get        115\n",
       "on         114\n",
       "Girl       114\n",
       "Never      113\n",
       "for        112\n",
       "World      112\n",
       "Do         111\n",
       "We         108\n",
       "It's       107\n",
       "Rock       105\n",
       "Day        104\n",
       "Little     104\n",
       "If         103\n",
       "Know       102\n",
       "&          102\n",
       "That       102\n",
       "Away       102\n",
       "dtype: int64"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "names = df_all.track_name.to_list()\n",
    "words = []\n",
    "for name in names:\n",
    "    words.extend(str(name).split(' '))\n",
    "    \n",
    "pd.Series(words).value_counts().head(60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "18b2fc9e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(16762, 26)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_all.to_csv('16k_songs_and_lyrics.csv', index=None)\n",
    "df_all.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2794090",
   "metadata": {},
   "source": [
    "### The second part of the Lyrics Entropy project is available [here](https://carlosfg97.github.io/MusicLyricEntropy/Music-Lyrics-Entropy.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a821589",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "17595bb4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20d6085e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83e75c3d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b13282ff",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "138603aa",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05835d08",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca964411",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f502d58",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "909c9cdf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9f2794c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7dd23a79",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "571ad753",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7cb2e43",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ad21ae13",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fc16c7d3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "987a6bae",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "319a12cc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0daf1b23",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcac8a29",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "965b9344",
   "metadata": {},
   "source": [
    "### ALBUM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5d3643fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "album_id = list(set( df_tracks.album_id.to_list()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "feec352c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_complete_album_info(album):\n",
    "\n",
    "    ab = sp.album(album)\n",
    "    album_artist =  ab['artists'][0]['id']\n",
    "    album_name =ab['name']\n",
    "    album_type = ab['album_type']\n",
    "    album_release_date = ab['release_date']\n",
    "    album_label = ab['label']\n",
    "       \n",
    "    output = {\n",
    "        'album_id' : album,\n",
    "        'album_artist' : album_artist,\n",
    "        'album_name' : album_name,\n",
    "        'album_type' : album_type,\n",
    "        'album_release_date' : album_release_date #,\n",
    "        #'album_label' : album_label\n",
    "    }\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "f596326a",
   "metadata": {},
   "outputs": [],
   "source": [
    "album_info = []\n",
    "for i in album_id:\n",
    "    album_info.append(get_complete_album_info(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "a3349802",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_album = pd.DataFrame(album_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c12df00d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_album.to_csv('albums.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f110f02",
   "metadata": {},
   "source": [
    "### PLAYLIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "7668ce6a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['532F1h299qKD894BlPfJJF',\n",
       " '7bVwBxsWk84iKeMPXa8oRs',\n",
       " '6dSeKJvN1MhLP1IEwBWFXg',\n",
       " '0f2CvetujnCTOxB6KCXhII',\n",
       " '37i9dQZF1DXcBWIGoYBM5M',\n",
       " '37i9dQZF1DX4bSrsRWE9cd',\n",
       " '37i9dQZF1DWU3bkMPOyjie',\n",
       " '37i9dQZF1DWY4xHQp97fN6',\n",
       " '37i9dQZF1DX4WYpdgoIcn6',\n",
       " '37i9dQZF1DX5NsFgylu4qQ',\n",
       " '37i9dQZF1DX82pCGH5USnM',\n",
       " '37i9dQZF1DWYs83FtTMQFw']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "playlist_ids "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "5c371c04",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_playlist_info(playlist_id):\n",
    "    \n",
    "    # basic info\n",
    "    p = sp.playlist(playlist_id)\n",
    "    flg_collaborative = p['collaborative']\n",
    "    playlist_description =  p['description']\n",
    "    playlist_followers =     p['followers']['total']\n",
    "    playlist_url =    p['href']\n",
    "    playlist_name =     p['name']\n",
    "    playlist_owner =    p['owner']['id']\n",
    "    flg_public =    p['public']\n",
    "\n",
    "    \n",
    "    output = {\n",
    "        'playlist_id' : playlist_id,\n",
    "        'flg_collaborative' : flg_collaborative,\n",
    "        'playlist_description' : playlist_description,\n",
    "        'playlist_followers' : playlist_followers,\n",
    "        'playlist_url' : playlist_url,\n",
    "        'playlist_name' : playlist_name,\n",
    "        'playlist_owner' : playlist_owner,\n",
    "        'flg_public' : flg_public,\n",
    "        \n",
    "    }\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4246c98d",
   "metadata": {},
   "outputs": [],
   "source": [
    "playlist_info = []\n",
    "\n",
    "for i in playlist_ids:\n",
    "    playlist_info.append(get_playlist_info(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "ce392af2",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_playlist = pd.DataFrame(playlist_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "b031cb91",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_playlist.to_clipboard(index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "905ce2bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_playlist_track_relation(playlist_id):\n",
    "    \n",
    "    pt = sp.playlist_items(playlist_id)['items']\n",
    "    \n",
    "    playlist_track_added_by = [i['added_by']['id'] for i in pt]\n",
    "    playlist_track_added_at =  [i['added_at'] for i in pt]\n",
    "    playlist_track_id = [i['track']['id'] for i in pt]\n",
    "    \n",
    "    output = {\n",
    "        'playlist_id' : playlist_id,\n",
    "        #'playlist_track_added_by' : playlist_track_added_by,\n",
    "        #'playlist_track_added_at' : playlist_track_added_at,\n",
    "        'track_id' : playlist_track_id   \n",
    "    }\n",
    "    \n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e0bb95dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_playlist_track  = pd.DataFrame(get_playlist_track_relation('37i9dQZF1DWYs83FtTMQFw'))[0:0]\n",
    "for i in df_playlist.playlist_id:\n",
    "    \n",
    "    df_playlist_track = pd.concat([df_playlist_track, \n",
    "                    pd.DataFrame(get_playlist_track_relation(i))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "60571ab9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_playlist_track = df_playlist_track.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "e632dfee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_playlist_track.to_clipboard(index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}