Vagrantfile

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
# 
#   http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# -*- mode: ruby -*-
# vi: set ft=ruby :

#
# Builds a single node Rya on Accumulo on Hadoop and Zookeeper.
# Deploys Rya, and the Eclipse rdf4j-workbench on Tomcat accessible from the host's browser.
# See the accompanying readme for URL's, verification, and troubleshooting.
#
# Note: Machine's ip is 192.168.33.10
# username : vagrant
# password : vagrant
#
# accumulo instance : dev
# accumulo username : root
# accumulo password : root

Vagrant.configure(2) do |config|

  config.vm.box = "bento/ubuntu-18.04"
  config.vm.provider "virtualbox" do |vb|
    vb.name = "rya-example-box"
    vb.memory = "4096"
    vb.cpus = 4
  end

  config.vm.network :private_network, ip: "192.168.33.10"
  config.vm.hostname = "rya-example-box"

  config.vm.provision "shell", inline: <<-SHELL
    set -x  ## turn on command echo with expanded variables
    # List of dependency versions
    export ACCUMULO_VERSION=1.6.6
    export HADOOP_VERSION=2.7.2
    ### IMPORTANT:  Verify this exists in the Maven repos.  As you know, I am writing this before it exists.
    export RYA_EXAMPLE_VERSION=4.0.1
    export RDF4J_VERSION=2.5.5
    export ZOOKEEPER_VERSION=3.4.5-cdh5.16.1
    mavenRepoUrl=https://repo1.maven.org/maven2/
    echo "Updating host file with permanent ip"
    sudo sed -i 's/127.0.1.1/192.168.33.10/' /etc/hosts
    cat >> /etc/hosts <<EOF
127.0.0.1     localhost
192.168.33.10 zoo1 zoo2 zoo3 leader1
EOF
    sudo -E apt-get -qq update
    echo "Installing Java installer..."
    # if you want dev tools like javac, change this to openjdk-8-jdk-headless 
    sudo -E apt-get install -y openjdk-8-jre || exit $?
    
    echo "Installing Tomcat..."
    sudo useradd -r -m -U -d /opt/tomcat -s /bin/false tomcat
    sudo -E apt-get install -y tomcat9  || exit $?
    sudo usermod -a -G tomcat vagrant
    echo "Installing Unzip..."
    apt-get install unzip || exit $?
    echo "Setting up environment..."
    export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/jre
    export HADOOP_HOME=/home/vagrant/hadoop-${HADOOP_VERSION}
    export ZOOKEEPER_HOME=/home/vagrant/zookeeper-${ZOOKEEPER_VERSION}
    export ZOO_LOG_DIR=${ZOOKEEPER_HOME}/logs/
    export ACCUMULO_HOME=/home/vagrant/accumulo-${ACCUMULO_VERSION}
    export PATHADD=$JAVA_HOME/bin:$ZOOKEEPER_HOME/bin:$ACCUMULO_HOME/bin:$HADOOP_HOME/bin
    export PATH=$PATH:$PATHADD
    export HADOOP_PREFIX="$HADOOP_HOME"
    export HADOOP_CONF_DIR="$HADOOP_PREFIX/etc/hadoop"
    export ACCUMULO_LOG_DIR=$ACCUMULO_HOME/logs
    export ACCUMULO_TSERVER_OPTS="-Xmx384m -Xms384m "
    export ACCUMULO_MASTER_OPTS="-Xmx128m -Xms128m"
    export ACCUMULO_MONITOR_OPTS="-Xmx64m -Xms64m"
    export ACCUMULO_GC_OPTS="-Xmx64m -Xms64m"
    export ACCUMULO_GENERAL_OPTS="-XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -Djava.net.preferIPv4Stack=true"
    export ACCUMULO_OTHER_OPTS="-Xmx128m -Xms64m"
    export ACCUMULO_KILL_CMD='kill -9 %p'
    # Shell environment includes Accumulo resources.
    ACCUMULO_RC=/home/vagrant/.accumulo_rc.sh
    cat > ${ACCUMULO_RC}  <<EOF
        export JAVA_HOME=\'$JAVA_HOME\'
        export HADOOP_HOME=\'$HADOOP_HOME\'
        export ZOOKEEPER_HOME=\'$ZOOKEEPER_HOME\'
        export ZOO_LOG_DIR=\'$ZOO_LOG_DIR\'
        export ACCUMULO_HOME=\'$ACCUMULO_HOME\'
        export PATH=\\$PATH:$PATHADD
        export HADOOP_PREFIX=\'$HADOOP_PREFIX\'
        export HADOOP_CONF_DIR=\'$HADOOP_CONF_DIR\'
        export ACCUMULO_LOG_DIR=\'$ACCUMULO_LOG_DIR\'
        export ACCUMULO_TSERVER_OPTS=\'$ACCUMULO_TSERVER_OPTS\'
        export ACCUMULO_MASTER_OPTS=\'$ACCUMULO_MASTER_OPTS\'
        export ACCUMULO_MONITOR_OPTS=\'$ACCUMULO_MONITOR_OPTS\'
        export ACCUMULO_GC_OPTS=\'$ACCUMULO_GC_OPTS\'
        export ACCUMULO_GENERAL_OPTS=\'$ACCUMULO_GENERAL_OPTS\'
        export ACCUMULO_OTHER_OPTS=\'$ACCUMULO_OTHER_OPTS\'
        export ACCUMULO_KILL_CMD=\'$ACCUMULO_KILL_CMD\'
        
        ### command to list the 7 correct java processes: tomcat-catalina, zookeeper, and 5 Accumulo: tracer, master, monitor, tserver, gc.
        function ryaps() { ps -ef | grep java | tr ' ' '\\n' | egrep '^org\\.apache|^tracer|^master|^monitor|^tserver|^gc' | sed '/\\.Main/ N ; s/\\n/ /' ; }
EOF
    source ${ACCUMULO_RC} || exit 151
    # include it at the beginning of both shell configuration files.
    for BASHRC in /home/vagrant/.bashrc /home/vagrant/.bash_profile ;
    do touch ${BASHRC} ;
        cat - ${BASHRC} > ${BASHRC}.new <<EOF  && mv ${BASHRC}.new ${BASHRC} || exit 152
source ${ACCUMULO_RC}
EOF
    done
    echo "Acquiring and Extracting ..."
    
    function echoerr() { printf "%s\n" "$*" >&2; }
    function download {
      ### curl --fail treat http status >= 400 as an error. --location follow redirects status>=300
      curl --silent --show-error --fail --location "$@"
      if [ $? -ne 0 ]; then
        echoerr "--------------------------"
        echoerr "-"
        echoerr "- download failed" "$@"
        echoerr "-"
        echoerr "-"   exiting ...
        echoerr "-"
        echoerr "--------------------------"
        exit 800
      fi
    }
    ### wait for a directory to exist or 60 seconds timeout
    function waitForDeploy {
        waitfordir="$1"
        timeout=60
        while [[ ! -d  "$waitfordir" ]]  
        do
            sleep 5
            let timeout-=5
            if [[ $timeout -le "0" ]]; then 
                echo "Timeout waiting for war to deploy, $waitfordir still does not exist."; 
                exit 401 
            fi
        done
    }
    echo "- Hadoop"
    hadoopUrl=https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
    if [[ ! -d ${HADOOP_HOME} ]] ; then 
        echo "Downloading $hadoopUrl"
        download $hadoopUrl \
        | tar -zxC /home/vagrant || exit 101
    fi
    echo "- Zookeeper"
    zookeeperUrl=http://archive.cloudera.com/cdh5/cdh/5/zookeeper-${ZOOKEEPER_VERSION}.tar.gz
    if [[ ! -d ${ZOOKEEPER_HOME} ]] ; then
        echo "Downloading $zookeeperUrl"
        download $zookeeperUrl \
        | tar -zxC /home/vagrant || exit 102
    fi
    echo "- Accumulo"
    accumuloUrl=https://archive.apache.org/dist/accumulo/${ACCUMULO_VERSION}/accumulo-${ACCUMULO_VERSION}-bin.tar.gz
    if [[ ! -d ${ACCUMULO_HOME} ]] ; then
        echo "Downloading $accumuloUrl"
        download $accumuloUrl \
        | tar -zxC /home/vagrant || exit 103
    fi
    echo "Configuring Zookeeper..."
    sudo mkdir --parents /var/zookeeper
    sudo chown vagrant:vagrant /var/zookeeper
    sudo cp ${ZOOKEEPER_HOME}/conf/zoo_sample.cfg ${ZOOKEEPER_HOME}/conf/zoo.cfg
    # Change the dataDir to /var/zookeeper -- strange that backslashes need to be escaped only on this line.
    sudo sed -i 's/^\\s*dataDir\\s*=.*$/dataDir=\\/var\\/zookeeper/' ${ZOOKEEPER_HOME}/conf/zoo.cfg  || exit 104
    # Conflicts with Accumulo and maybe Zookeeper
    sudo rm --force ${HADOOP_HOME}/share/hadoop/common/lib/slf4j-api-1.7.10.jar
    sudo rm --force ${HADOOP_HOME}/share/hadoop/common/lib/slf4j-log4j12-1.7.10.jar
    # Assure logs are creatable and writeable
    sudo mkdir --parents ${ZOO_LOG_DIR}
    sudo touch "${ZOO_LOG_DIR}/zookeeper.out"
    sudo chmod -R a+wX  ${ZOO_LOG_DIR}
    echo "Running Zookeeper..."
    sudo -E ${ZOOKEEPER_HOME}/bin/zkServer.sh start
    echo "Configuring Accumulo..."
    cp ${ACCUMULO_HOME}/conf/examples/1GB/standalone/* ${ACCUMULO_HOME}/conf/
    rm --force ${ACCUMULO_HOME}/conf/accumulo-site.xml
    cat >> ${ACCUMULO_HOME}/conf/accumulo-site.xml <<EOF
        <configuration>
            <property><name>instance.dfs.uri</name><value>file:///</value></property>
            <property><name>instance.dfs.dir</name><value>/data/accumulo</value></property>
            <property><name>instance.zookeeper.host</name><value>localhost:2181</value></property>
            <property><name>instance.secret</name><value>DONTTELL</value></property>
            <property><name>tserver.port.search</name><value>true</value></property>
            <property><name>logger.dir.walog</name><value>/data/accumulo/walogs</value></property>
            <property><name>tserver.cache.data.size</name><value>15M</value></property>
            <property><name>tserver.cache.index.size</name><value>15M</value></property>
            <property><name>tserver.memory.maps.max</name><value>256M</value></property>
            <property><name>tserver.walog.max.size</name><value>256M</value></property>
            <property><name>tserver.memory.maps.native.enabled</name><value>false</value></property>
            <property><name>trace.token.property.password</name><value>root</value></property>
            <property><name>gc.cycle.delay</name><value>4s</value></property>
            <property><name>gc.cycle.start</name><value>0s</value></property>
            <property><name>tserver.compaction.major.delay</name><value>3</value></property>
            <property><name>general.classpaths</name><value>
            /data/accumulo/lib/[^.].*.jar,
            ${HADOOP_HOME}/share/hadoop/common/.*.jar,
            ${HADOOP_HOME}/share/hadoop/common/lib/.*.jar,
            ${HADOOP_HOME}/share/hadoop/hdfs/.*.jar,
            ${HADOOP_HOME}/share/hadoop/mapreduce/.*.jar,
            ${HADOOP_HOME}/share/hadoop/yarn/.*.jar,
            ${ACCUMULO_HOME}/server/target/classes/,
            ${ACCUMULO_HOME}/lib/accumulo-server.jar,
            ${ACCUMULO_HOME}/core/target/classes/,
            ${ACCUMULO_HOME}/lib/accumulo-core.jar,
            ${ACCUMULO_HOME}/start/target/classes/,
            ${ACCUMULO_HOME}/lib/accumulo-start.jar,
            ${ACCUMULO_HOME}/fate/target/classes/,
            ${ACCUMULO_HOME}/lib/accumulo-fate.jar,
            ${ACCUMULO_HOME}/proxy/target/classes/,
            ${ACCUMULO_HOME}/lib/accumulo-proxy.jar,
            ${ACCUMULO_HOME}/lib/[^.].*.jar,
            /home/vagrant/${ZOOKEEPER_HOME}/zookeeper[^.].*.jar,
            $HADOOP_CONF_DIR,
            ${HADOOP_HOME}/[^.].*.jar,
            ${HADOOP_HOME}/lib/[^.].*.jar,
            </value></property>
            <property><name>general.dynamic.classpaths</name><value>/data/accumulo/lib/ext/[^.].*.jar</value></property>
            <property><name>trace.port.client</name><value>0</value></property>
            <property><name>monitor.port.client</name><value>0</value></property>
            <property><name>master.port.client</name><value>0</value></property>
            <property><name>tserver.port.client</name><value>0</value></property>
            <property><name>gc.port.client</name><value>0</value></property>
        </configuration>
EOF
    cat > ${ACCUMULO_HOME}/conf/masters <<EOF
rya-example-box
EOF
    cat > ${ACCUMULO_HOME}/conf/slaves <<EOF
rya-example-box
EOF
    sudo mkdir --parents /data
    sudo chown vagrant:vagrant /data
    mkdir --parents /data/accumulo/lib/ext
    
    sudo chmod -R a+rwX ${ACCUMULO_HOME}/logs/
    echo "Starting Accumulo..."
    echo "Init will fail during a re-provision, but you can ignore it:  'FATAL: It appears the directories [...] were previously initialized.'"
    echo "Also, you may see an indefinitely repeating: 'Waiting for accumulo to be initialized' which means Accumulo won't start."
    echo "Either issue can be resolved by removing the directory: 'sudo rm -r /data/accumulo' then re-provision. Warning: this will erase all Rya/Accumulo data."
    ${ACCUMULO_HOME}/bin/accumulo init --instance-name dev --password root
    ${ACCUMULO_HOME}/bin/start-all.sh  || exit 107
    sudo chmod -R a+rwX ${ACCUMULO_HOME}/logs/
    echo 'Done!'
    # --------------------
    echo "Installing RDF4J Server"
    # this is the rdf4j data dir.  It might need to be registered with tomcat to write there.
    export rdf4jServerBase=/opt/tomcat/
    sudo install -d -o tomcat -m u=rwx,go=rx ${rdf4jServerBase}
    sudo -u tomcat mkdir --parents ${rdf4jServerBase}/server/logs
    sudo -u tomcat mkdir --parents ${rdf4jServerBase}/.RDF4J/server/logs
    # tomcat v9 requires explicit permissions to the file system.
    sudo mkdir --parents          /etc/systemd/system/tomcat9.service.d
    cat <<EOF | sudo tee --append /etc/systemd/system/tomcat9.service.d/logging-allow.conf
[Service]
ReadWritePaths=${rdf4jServerBase}
EOF
    echo 'Restarting Tomcat (or starting)'
    sudo -E systemctl daemon-reload     || exit 1080
    sudo -E systemctl enable tomcat9    || exit 1081
    sudo -E systemctl restart tomcat9   || exit 1082
    rdf4jwar=/var/lib/tomcat9/webapps/rdf4j-server.war
    if [[ ! -s $rdf4jwar ]] ; then
        echo "Downloading RDF4J Server"
        download --output $rdf4jwar ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-server/${RDF4J_VERSION}/rdf4j-http-server-${RDF4J_VERSION}.war || exit 110
    fi
    echo "RDF4J http server deployed at http://rya-example-box:8080/rdf4j-server"
    # --------------------
    echo "Installing RDF4J Workbench"
    workbench=/var/lib/tomcat9/webapps/rdf4j-workbench.war
    if [[ ! -s $workbench ]] ; then
        echo "Downloading RDF4J Workbench"
        download --output $workbench ${mavenRepoUrl}org/eclipse/rdf4j/rdf4j-http-workbench/${RDF4J_VERSION}/rdf4j-http-workbench-${RDF4J_VERSION}.war || exit 111
    fi
    echo "RDF4J workbench deployed at http://rya-example-box:8080/rdf4j-workbench"
    # ----------------------
    echo "Installing Rya"
    ryaIndexing=rya.indexing.example-${RYA_EXAMPLE_VERSION}-distribution
    # Place in the vagrant folder from your dev project, example: extras/indexingExample/target/rya.indexing.example-4.0.1-distribution.zip
    if [[ -s "/vagrant/${ryaIndexing}.zip" ]] ; then
        echo "Using found local file:  /vagrant/${ryaIndexing}.zip "
        cp "/vagrant/${ryaIndexing}.zip"  ./
    fi
    if [[ ! -s ${ryaIndexing}.zip ]] ; then
        echo "Downloading ${ryaIndexing}.zip quietly, this will take some minutes with no output..."
        download --output ${ryaIndexing}.zip ${mavenRepoUrl}org/apache/rya/rya.indexing.example/${RYA_EXAMPLE_VERSION}/${ryaIndexing}.zip || exit 112
    fi
    sudo mkdir --parents ${ryaIndexing}
    sudo unzip -q -o ${ryaIndexing}.zip -d ${ryaIndexing}
    # before continuing, wait for tomcat to deploy wars:
    waitForDeploy /var/lib/tomcat9/webapps/rdf4j-workbench/WEB-INF/lib/
    waitForDeploy /var/lib/tomcat9/webapps/rdf4j-server/WEB-INF/lib/
    # soft linking the files doesn't seem to work in tomcat, so we copy them instead :(
    sudo -u tomcat cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat9/webapps/rdf4j-workbench/WEB-INF/lib/ || exit 113
    sudo -u tomcat cp ${ryaIndexing}/dist/lib/* /var/lib/tomcat9/webapps/rdf4j-server/WEB-INF/lib/    || exit 114
    # These are older libs that break tomcat and rdf4j that come with Rya above.
    sudo rm --force /var/lib/tomcat9/webapps/rdf4j-workbench/WEB-INF/lib/servlet-api-2.5.jar
    sudo rm --force /var/lib/tomcat9/webapps/rdf4j-workbench/WEB-INF/lib/jsp-api-2.1.jar
    s=/var/lib/tomcat9/webapps/rdf4j-server/WEB-INF/lib/
    sudo rm --force $s/servlet-api-2.5.jar   $s/lib/jsp-api-2.1.jar
    sudo find "$s" -name 'spring-*-3.*.jar' -exec sudo rm --force {} +
    sudo rm $s/spring-aop-4.2.1.RELEASE.jar $s/rdf4j-http-server-spring-2.3.1.jar
    sudo rm --force  $s/slf4j-log4j12-1.7.25.jar   $s/slf4j-api-1.7.25.jar  $s/jcabi-log-0.14.jar   $s/jcl-over-slf4j-1.7.25.jar   $s/minlog-1.3.0.jar   $s/commons-logging-1.1.1.jar   $s/log4j-1.2.16.jar
    sudo chown -R tomcat:tomcat /var/lib/tomcat9/webapps/rdf4j-workbench/WEB-INF/lib/
    sudo chown -R tomcat:tomcat /var/lib/tomcat9/webapps/rdf4j-server/WEB-INF/lib/
    # ----------------------
    echo "Downloading and installing new templates for RDF4J WorkBench"
    ryaVagrant=rya.vagrant.example-${RYA_EXAMPLE_VERSION}
    # Place in the vagrant folder from your dev project: extras/vagrantExample/target/rya.vagrant.example-*.jar 
    if [[ -s "/vagrant/${ryaVagrant}.jar" ]] ; then
        echo "Using found local file:  /vagrant/${ryaVagrant}.jar "
        cp "/vagrant/${ryaVagrant}.jar"  ./
    fi
    if [[ ! -s ${ryaVagrant}.jar ]] ; then
        echo "Downloading ${ryaVagrant}.jar"
        download --output ${ryaVagrant}.jar ${mavenRepoUrl}org/apache/rya/rya.vagrant.example/${RYA_EXAMPLE_VERSION}/${ryaVagrant}.jar || exit 120
    fi
    sudo mkdir --parents ${ryaVagrant}
    sudo unzip -q -o ${ryaVagrant}.jar -d ${ryaVagrant}
    waitForDeploy /var/lib/tomcat9/webapps/rdf4j-workbench/transformations
    sudo -u tomcat cp ${ryaVagrant}/*.xsl /var/lib/tomcat9/webapps/rdf4j-workbench/transformations/
    sudo chown tomcat:tomcat /var/lib/tomcat9/webapps/rdf4j-workbench/transformations/*
    # ----------------------
    echo "Deploying Rya Web"
    ryaWar=web.rya-${RYA_EXAMPLE_VERSION}.war
    # Place in the vagrant folder from your dev project: web/web.rya/target/web.rya.war  rename to add version.
    if [[ -s "/vagrant/${ryaWar}" ]] ; then
        echo "Using found local file: /vagrant/${ryaWar} "
        cp "/vagrant/${ryaWar}"  ./
    fi
    if [[ ! -s ${ryaWar} ]] ; then
        echo "Downloading ${ryaWar}"
        ###download https://www.dropbox.com/s/jrmgfey8ch1vrd6/${ryaWar}?raw=1 --output ${ryaWar} || exit 121
        download ${mavenRepoUrl}org/apache/rya/web.rya/${RYA_EXAMPLE_VERSION}/${ryaWar} --output ${ryaWar} || exit 121
    fi
    # remove this so we can wait for it to be recreated.
    sudo rm -rf /var/lib/tomcat9/webapps/web.rya/WEB-INF/classes
    sudo -u tomcat cp    ${ryaWar} /var/lib/tomcat9/webapps/web.rya.war
    # Wait for the war to deploy
    waitForDeploy /var/lib/tomcat9/webapps/web.rya/WEB-INF/classes/
    # These are older libs that break tomcat
    sudo rm --force /var/lib/tomcat9/webapps/web.rya/WEB-INF/lib/servlet-api-2.5*.jar
    sudo rm --force /var/lib/tomcat9/webapps/web.rya/WEB-INF/lib/jsp-api-2.1.jar
    echo "Modify Rya Web Config"
    sudo chmod -R a+rwX /var/lib/tomcat9/webapps/web.rya/
    cat > /var/lib/tomcat9/webapps/web.rya/WEB-INF/classes/environment.properties <<EOF
instance.name=dev
instance.zk=localhost:2181
instance.username=root
instance.password=root
rya.tableprefix=rya_
rya.displayqueryplan=true
EOF
    echo "Rya web deployed at http://rya-example-box:8080/web.rya/sparqlQuery.jsp"
    # restart tomcat
    systemctl restart tomcat9   || exit $?
    echo "Finished and ready to use!"
    echo "You can re-apply these settings without losing data by running the command 'vagrant provision'"
  SHELL
end