sparklyR.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Attaching package: ‘dplyr’\n",
      "\n",
      "The following objects are masked from ‘package:stats’:\n",
      "\n",
      "    filter, lag\n",
      "\n",
      "The following objects are masked from ‘package:base’:\n",
      "\n",
      "    intersect, setdiff, setequal, union\n",
      "\n"
     ]
    }
   ],
   "source": [
    "library(dplyr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ERROR",
     "evalue": "Error in library(sparklyr): there is no package called ‘sparklyr’\n",
     "output_type": "error",
     "traceback": [
      "Error in library(sparklyr): there is no package called ‘sparklyr’\nTraceback:\n",
      "1. library(sparklyr)",
      "2. stop(txt, domain = NA)"
     ]
    }
   ],
   "source": [
    "library(ggplot2)\n",
    "library(sparklyr)\n",
    "source(\"Rstart.R\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "also installing the dependencies ‘httpuv’, ‘xtable’, ‘sourcetools’, ‘later’, ‘promises’, ‘config’, ‘rappdirs’, ‘shiny’\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The downloaded binary packages are in\n",
      "\t/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T//Rtmpo81QXZ/downloaded_packages\n"
     ]
    }
   ],
   "source": [
    "install.packages(\"sparklyr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "library(sparklyr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading GitHub repo rstudio/sparklyr@master\n",
      "from URL https://api.github.com/repos/rstudio/sparklyr/zipball/master\n",
      "Installing sparklyr\n",
      "Installing digest\n",
      "'/Library/Frameworks/R.framework/Resources/bin/R' --no-site-file --no-environ  \\\n",
      "  --no-save --no-restore --quiet CMD INSTALL  \\\n",
      "  '/private/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T/Rtmpo81QXZ/devtools817f128362e1/digest'  \\\n",
      "  --library='/Library/Frameworks/R.framework/Versions/3.5/Resources/library'  \\\n",
      "  --install-tests \n",
      "\n",
      "Installing r2d3\n",
      "Installing htmlwidgets\n",
      "'/Library/Frameworks/R.framework/Resources/bin/R' --no-site-file --no-environ  \\\n",
      "  --no-save --no-restore --quiet CMD INSTALL  \\\n",
      "  '/private/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T/Rtmpo81QXZ/devtools817f1e688124/htmlwidgets'  \\\n",
      "  --library='/Library/Frameworks/R.framework/Versions/3.5/Resources/library'  \\\n",
      "  --install-tests \n",
      "\n",
      "'/Library/Frameworks/R.framework/Resources/bin/R' --no-site-file --no-environ  \\\n",
      "  --no-save --no-restore --quiet CMD INSTALL  \\\n",
      "  '/private/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T/Rtmpo81QXZ/devtools817f75c7db5/r2d3'  \\\n",
      "  --library='/Library/Frameworks/R.framework/Versions/3.5/Resources/library'  \\\n",
      "  --install-tests \n",
      "\n",
      "Downloading GitHub repo kevinykuo/forge@master\n",
      "from URL https://api.github.com/repos/kevinykuo/forge/zipball/master\n",
      "Installing forge\n",
      "'/Library/Frameworks/R.framework/Resources/bin/R' --no-site-file --no-environ  \\\n",
      "  --no-save --no-restore --quiet CMD INSTALL  \\\n",
      "  '/private/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T/Rtmpo81QXZ/devtools817f8b9c9c3/rstudio-forge-f84b834'  \\\n",
      "  --library='/Library/Frameworks/R.framework/Versions/3.5/Resources/library'  \\\n",
      "  --install-tests \n",
      "\n",
      "'/Library/Frameworks/R.framework/Resources/bin/R' --no-site-file --no-environ  \\\n",
      "  --no-save --no-restore --quiet CMD INSTALL  \\\n",
      "  '/private/var/folders/nw/3fnxvmmj6lb4vv4_d_jr__v40000gn/T/Rtmpo81QXZ/devtools817f1e10707d/rstudio-sparklyr-cac2761'  \\\n",
      "  --library='/Library/Frameworks/R.framework/Versions/3.5/Resources/library'  \\\n",
      "  --install-tests \n",
      "\n",
      "Reloading installed sparklyr\n"
     ]
    }
   ],
   "source": [
    "devtools::install_github(\"rstudio/sparklyr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config <- spark_config()\n",
    "config$`sparklyr.shell.driver-memory` <- \"8G\"\n",
    "config$spark.local.dir <- \"/Volumes/MAX128GB/tmp\"\n",
    "# Spark 2.1.0 was released but not supported by sparklyr at time of writing\n",
    "sc <- spark_connect(master = \"local\", config = config, version=\"2.0.2\")\n",
    "sc$config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark_install(version = \"2.0.2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path <- \"/Users/clarencestephen/github/Projects/05-kojak/data\"\n",
    "# do not cache the initial data set to save memory; only cache the final result\n",
    "df <- spark_read_json(sc, 'data', paste(file_path, \"reviews_Books.json\", sep=\"/\"), header=T, overwrite=T, memory=F)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}