Fixing create-venv issues with arm

synesthesiam · May 21, 2019 · 7d46978 · 7d46978
1 parent 7359c3e
commit 7d46978
Show file tree

Hide file tree

Showing 2 changed files with 57 additions and 53 deletions.
diff --git a/create-venv.sh b/create-venv.sh
@@ -47,30 +47,26 @@ sudo apt-get install -y python3 python3-pip python3-venv python3-dev \
      gfortran \
      sphinxbase-utils sphinxtrain pocketsphinx \
      jq checkinstall unzip xz-utils \
-     curl
+     curl libffi-dev
 
 # Download dependencies
 echo "Downloading dependencies"
-bash download-dependencies.sh
+bash download-dependencies.sh "${CPU_ARCH}"
 
 # -----------------------------------------------------------------------------
 # OpenFST
 # -----------------------------------------------------------------------------
 
 case $CPU_ARCH in
     armv7l|arm64v8)
-        # Build from source
-        openfst_file="${download_dir}/openfst-1.6.2.tar.gz"
-        echo "Building OpenFST (${openfst_file})"
-        tar -C "${temp_dir}" -xzf "${openfst_file}" && \
-            cd "${temp_dir}/openfst-1.6.2" && \
-            ./configure --enable-static --enable-shared --enable-far --enable-ngram-fsts && \
-            make -j 4 && \
-            sudo make install
+        # Use pre-built packages
+        openfst_file="${download_dir}/openfst_1.6.9-1_${FRIENDLY_ARCH}.deb"
+        echo "Installing OpenFST (${openfst_file})"
+        sudo dpkg -i "${openfst_file}"
         ;;
 
     *)
-        # Use pre-built packages
+        # Use system packages
         sudo apt-get install -y libfst-dev libfst-tools
 esac
 
@@ -121,6 +117,9 @@ source "${VENV_PATH}/bin/activate"
 
 case $CPU_ARCH in
     armv7l|arm64v8)
+        # Force newer openfst
+        "${PYTHON}" -m pip install 'openfst==1.6.9'
+
         # Exclude flair
         grep -v flair requirements.txt > "${temp_dir}/requirements-noflair.txt"
         "${PYTHON}" -m pip install -r "${temp_dir}/requirements-noflair.txt"
@@ -181,14 +180,25 @@ fi
 # -----------------------------------------------------------------------------
 
 if [[ -z "$(which ngramcount)" ]]; then
-    opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz"
-    echo "Building Opengrm ${opengrm_file}"
-    tar -C "${temp_dir}" -xf "${opengrm_file}" && \
-        cd "${temp_dir}/opengrm-ngram-1.3.3" && \
-        ./configure && \
-        make -j 4 && \
-        sudo make install && \
-        sudo ldconfig
+    case $CPU_ARCH in
+        armv7l|arm64v8)
+            # Use pre-built packages
+            opengrm_file="${download_dir}/opengrm_1.3.4-1_${FRIENDLY_ARCH}.deb"
+            echo "Installing opengrm (${opengrm_file})"
+            sudo dpkg -i "${opengrm_file}"
+            ;;
+
+        *)
+            # Build from source
+            opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz"
+            echo "Building Opengrm ${opengrm_file}"
+            tar -C "${temp_dir}" -xf "${opengrm_file}" && \
+                cd "${temp_dir}/opengrm-ngram-1.3.3" && \
+                ./configure && \
+                make -j 4 && \
+                sudo make install && \
+                sudo ldconfig
+    esac
 fi
 
 # -----------------------------------------------------------------------------
@@ -219,30 +229,6 @@ fi
 # Add /usr/local/lib to LD_LIBRARY_PATH
 sudo ldconfig
 
-# -----------------------------------------------------------------------------
-# NodeJS / Yarn
-# -----------------------------------------------------------------------------
-
-if [[ -z "$(which node)" ]]; then
-    echo "Installing nodejs"
-    sudo apt-get install -y nodejs
-fi
-
-if [[ -z "$(which yarn)" ]]; then
-    echo "Installing yarn"
-    curl -o- -L https://yarnpkg.com/install.sh | bash
-
-    # Need to re-source .bashrc so yarn is in the path
-    source "${HOME}/.bashrc"
-fi
-
-# -----------------------------------------------------------------------------
-# Web Interface
-# -----------------------------------------------------------------------------
-
-echo "Building web interface"
-cd "${DIR}" && yarn && yarn build
-
 # -----------------------------------------------------------------------------
 
 echo "Done"
diff --git a/download-dependencies.sh b/download-dependencies.sh
@@ -8,25 +8,32 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 download_dir="${DIR}/download"
 mkdir -p "${download_dir}"
 
-# CPU architecture
-CPU_ARCHS=("x86_64" "armv7l" "arm64v8")
-FRIENDLY_ARCHS=("amd64" "armhf" "aarch64")
-
 declare -A CPU_TO_FRIENDLY
 CPU_TO_FRIENDLY["x86_64"]="amd64"
 CPU_TO_FRIENDLY["armv7l"]="armhf"
 CPU_TO_FRIENDLY["arm64v8"]="aarch64"
 
+# CPU architecture
+CPU_ARCHS=("x86_64" "armv7l" "arm64v8")
+FRIENDLY_ARCHS=("amd64" "armhf" "aarch64")
+
+if [[ ! -z "$1" ]]; then
+    CPU_ARCHS=("$1")
+    FRIENDLY_ARCHS=(${CPU_TO_FRIENDLY["$1"]})
+fi
+
 # -----------------------------------------------------------------------------
 # OpenFST
 # -----------------------------------------------------------------------------
 
-openfst_file="${download_dir}/openfst-1.6.2.tar.gz"
-if [[ ! -f "${openfst_file}" ]]; then
-    openfst_url='http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-1.6.2.tar.gz'
-    echo "Downloading OpenFST source (${openfst_url})"
-    curl -sSfL -o "${openfst_file}" "${openfst_url}"
-fi
+for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"; do
+    openfst_file="${download_dir}/openfst_1.6.9-1_${FRIENDLY_ARCH}.deb"
+    if [[ ! -f "${openfst_file}" ]]; then
+        openfst_url="https://github.com/synesthesiam/docker-opengrm/releases/download/v1.3.4-${FRIENDLY_ARCH}/openfst_1.6.9-1_${FRIENDLY_ARCH}.deb"
+        echo "Downloading OpenFST pre-built binary (${openfst_url})"
+        curl -sSfL -o "${openfst_file}" "${openfst_url}"
+    fi
+done
 
 # -----------------------------------------------------------------------------
 # Pocketsphinx for Python
@@ -80,12 +87,23 @@ done
 # -----------------------------------------------------------------------------
 
 if [[ -z "$(which ngramcount)" ]]; then
+    # Download source
     opengrm_file="${download_dir}/opengrm-ngram-1.3.3.tar.gz"
     if [[ ! -f "${opengrm_file}" ]]; then
         opengrm_url='https://www.opengrm.org/twiki/pub/GRM/NGramDownload/opengrm-ngram-1.3.3.tar.gz'
         echo "Download Opengrm (${opengrm_url})"
         curl -sSfLk -o "${opengrm_file}" "${opengrm_url}"
     fi
+
+    # Download pre-built packages
+    for FRIENDLY_ARCH in "${FRIENDLY_ARCHS[@]}"; do
+        opengrm_file="${download_dir}/opengrm_1.3.4-1_${FRIENDLY_ARCH}.deb"
+        if [[ ! -f "${opengrm_file}" ]]; then
+            opengrm_url="https://github.com/synesthesiam/docker-opengrm/releases/download/v1.3.4-${FRIENDLY_ARCH}/opengrm_1.3.4-1_${FRIENDLY_ARCH}.deb"
+            echo "Downloading opengrm pre-built binary (${opengrm_url})"
+            curl -sSfL -o "${opengrm_file}" "${opengrm_url}"
+        fi
+    done
 fi
 
 # -----------------------------------------------------------------------------