ARM Backend using ruy for fp32 and int8

.github/workflows/arm.yml

@@ -0,0 +1,143 @@
+name: ARM
+'on':
+  push:
+    branches:
+    - main
+    - ci-sandbox
+  pull_request:
+    branches:
+    - '**'
+env:
+  ccache_basedir: ${{ github.workspace }}
+  ccache_dir: "${{ github.workspace }}/.ccache"
+  ccache_compilercheck: content
+  ccache_compress: 'true'
+  ccache_compresslevel: 9
+  ccache_maxsize: 200M
+  ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache
+  ndk: "${{ github.workspace }}/android-ndk-r23b"
+  abi: "arm64-v8a"
+  minsdk_version : 28
+  android_platform: 28
+
+jobs:
+  ubuntu:
+    name: "arm-v8a cross-compile via Android NDK"
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Install prerequisites
+      run: |
+          wget -c --quiet https://dl.google.com/android/repository/android-ndk-r23b-linux.zip
+          unzip -qq android-ndk-r23b-linux.zip
+          sudo apt-get -y install ccache cmake
+
+    - name: Generate ccache_vars for ccache based on machine
+      shell: bash
+      id: ccache_vars
+      run: |-
+        echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})"
+        echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')"
+
+    - name: Cache-op for build-cache through ccache
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.ccache_dir }}
+        key: ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }}
+        restore-keys: |-
+          ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}
+          ccache-${{ matrix.identifier }}-${{ steps.ccache_vars.outputs.hash }}
+          ccache-${{ matrix.identifier }}
+
+    - name: ccache environment setup
+      run: |-
+        echo "CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }}" >> $GITHUB_ENV
+        echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV
+        echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV
+        echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV
+        echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV
+
+    - name: ccache prolog
+      run: |-
+        ccache -s # Print current cache stats
+        ccache -z # Zero cache entry
+
+    - name: Generate buildfiles for marian on android via cmake
+      run: |-
+        mkdir -p build 
+        cd build
+        NDK=${{ env.ndk }}
+        ABI=${{ env.abi }}
+        MINSDK_VERSION=${{ env.minsdk_version }}
+        ANDROID_PLATFORM=${{ env.android_platform }}
+        OTHER_ANDROID_ARGS=(
+            -DANDROID_ARM_NEON=TRUE
+        )
+        OTHER_MARIAN_ARGS=(
+            -DCOMPILE_CUDA=off
+            -DCOMPILE_CPU=on
+            -DCMAKE_HAVE_THREADS_LIBRARY=1
+            -DCMAKE_USE_WIN32_THREADS_INIT=0
+            -DCMAKE_USE_PTHREADS_INIT=1
+            -DTHREADS_PREFER_PTHREAD_FLAG=ON
+            -DBUILD_ARCH=armv8-a
+            -DUSE_INTGEMM=off
+            -DUSE_SIMDE=on
+            -DUSE_RUY=on
+            -DUSE_ONNX_SGEMM=on # For time being.
+            # -DCOMPILE_WITHOUT_EXCEPTIONS=on # Apparently this can reduce the binary size, let's see.
+        )
+        # Additionally list variables finally configured.
+        cmake -L \
+            -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \
+            -DANDROID_TOOLCHAIN=clang \
+            -DANDROID_ABI=$ABI \
+            -DANDROID_PLATFORM=$ANDROID_PLATFORM \
+            -DANDROID_NATIVE_API_LEVEL=$MINSDKVERSION \
+            -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi-4.8 \
+            -DANDROID_STL=c++_static \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            "${OTHER_ANDROID_ARGS[@]}" "${OTHER_MARIAN_ARGS[@]}" \
+            ..
+
+
+    - name : Build marian for android
+      working-directory: build
+      run: |-
+          # Only build marian (lib) for now.
+          make -j2 
+
+    - name: ccache epilog
+      run: 'ccache -s # Print current cache stats'
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: ${{github.workspace}}/build/marian-decoder
+
+
+  # Disable release for now.
+  # release:
+  #   name: Release Latest Build
+  #   runs-on: ubuntu-latest
+  #   needs: [ubuntu]
+  #   if: github.ref == 'refs/heads/master'
+  #   steps:
+  #    - name: Download artifacts
+  #      uses: actions/download-artifact@v2
+  #     
+  #    - name: Update GitHub prerelease
+  #      uses: marvinpinto/action-automatic-releases@latest
+  #      with:
+  #        repo_token: ${{ secrets.GITHUB_TOKEN }}
+  #        automatic_release_tag: latest
+  #        prerelease: true
+  #        title: "Latest Build"
+  #        files: |
+  #          artifact/marian-decoder

.gitmodules

@@ -23,3 +23,9 @@
 [submodule "src/3rd_party/onnxjs"]
 	path = src/3rd_party/onnxjs
 	url = https://github.com/abhi-agg/onnxjs.git
+[submodule "src/3rd_party/simde-no-tests"]
+	path = src/3rd_party/simde-no-tests
+	url = https://github.com/simd-everywhere/simde-no-tests/
+[submodule "src/3rd_party/ruy"]
+	path = src/3rd_party/ruy
+	url = https://github.com/google/ruy

CMakeLists.txt

@@ -31,40 +31,43 @@ option(USE_CCACHE "Use ccache compiler cache (https://ccache.dev)" OFF)
 option(USE_CUDNN "Use CUDNN library" OFF)
 option(USE_DOXYGEN "Build documentation with Doxygen" ON)
 option(USE_FBGEMM "Use FBGEMM" OFF)
+option(USE_INTGEMM "Use INTGEMM" ON)
 option(USE_MKL "Compile with MKL support" ON)
 option(USE_MPI "Use MPI library" OFF)
 option(USE_NCCL "Use NCCL library" ON)
 option(USE_SENTENCEPIECE "Download and compile SentencePiece" ON)
 option(USE_STATIC_LIBS "Link statically against non-system libs" OFF)
 option(GENERATE_MARIAN_INSTALL_TARGETS "Generate Marian install targets (requires CMake 3.12+)" OFF)
-option(M32_BINARIES "Generate 32bit binaries even when building outside of WASM. Useful for testing some WASM specific functionality without the need for the compiling to WASM." OFF)
 option(COMPILE_WASM "Compile (wasm compatible) marian for WASM target" OFF)
 option(USE_WASM_COMPATIBLE_SOURCE "Enable the minimal marian sources that compile to wasm. Useful for debugging wasm failures by building same sources natively" OFF)
 
+option(USE_SIMDE "Enable simde to target instruction sets" OFF)
+option(USE_ONNX_SGEMM "Compile with wasm compatible blas" OFF)
+option(COMPILE_WITHOUT_EXCEPTIONS "Compile without exceptions" OFF)
+
 # cmake options that are dependent on USE_WASM_COMPATIBLE_SOURCE cmake option
 CMAKE_DEPENDENT_OPTION(USE_THREADS "Compile with multi-threading support" OFF
                        "USE_WASM_COMPATIBLE_SOURCE" ON)
-CMAKE_DEPENDENT_OPTION(USE_WASM_COMPATIBLE_BLAS "Compile with wasm compatible blas" ON
-                       "USE_WASM_COMPATIBLE_SOURCE" OFF)
-CMAKE_DEPENDENT_OPTION(COMPILE_WITHOUT_EXCEPTIONS "Compile without exceptions" ON
-                       "USE_WASM_COMPATIBLE_SOURCE" OFF)
 
 if (USE_WASM_COMPATIBLE_SOURCE)
   set(SPM_BUILD_LIBRARY_ONLY ON CACHE BOOL "Build only sentencepiece library (skip building executables)")
   add_compile_definitions(WASM_COMPATIBLE_SOURCE)
   # Setting USE_SSE2 definition to enable SSE2 specific code in "3rd_party/sse_mathfun.h" for wasm builds
   add_compile_definitions(USE_SSE2)
+  set(USE_ONNX_SGEMM ON CACHE BOOL "")
+  set(COMPILE_WITHOUT_EXCEPTIONS ON CACHE BOOL "")
 endif()
 
 if (COMPILE_WASM)
   add_compile_definitions(WASM)
   set(WORMHOLE ON CACHE BOOL "Use WASM wormhole in intgemm https://bugzilla.mozilla.org/show_bug.cgi?id=1672160")
 endif()
 
-if(M32_BINARIES OR COMPILE_WASM)
+
+if(COMPILE_WASM)
     set("BUILD_WIDTH" "-m32")
-else(M32_BINARIES OR COMPILE_WASM)
-    set("BUILD_WIDTH" "-m64")
+else(COMPILE_WASM)
+    set("BUILD_WIDTH" "")
 endif()
 
 if(NOT COMPILE_WASM)
@@ -167,6 +170,7 @@ if(MSVC)
   # set(INTRINSICS "/arch:AVX")
   add_definitions(-DUSE_SSE2=1)
 
+
   # Or maybe use these?
   set(INTRINSICS ${MSVC_BUILD_ARCH})
   # set(INTRINSICS "/arch:AVX512")
@@ -193,6 +197,16 @@ if(MSVC)
     set(EXT_LIBS ${EXT_LIBS} fbgemm)
     add_definitions(-DUSE_FBGEMM=1 -DFBGEMM_STATIC=1)
   endif(USE_FBGEMM)
+
+  if(USE_INTGEMM)
+    add_definitions(-DUSE_INTGEMM=1)
+  endif(USE_INTGEMM)
+
+  if(USE_SIMDE)
+    add_definitions(-DUSE_SIMDE=1)
+    add_definitions(-DSIMDE_ENABLE_NATIVE_ALIASES=1)
+  endif(USE_SIMDE)
+
 else(MSVC)
 
   # Check we are using at least g++ 5.0
@@ -201,14 +215,13 @@ else(MSVC)
   endif()
 
   # Detect support CPU instrinsics for the current platform. This will
-  # only by used with BUILD_ARCH=native. For overridden BUILD_ARCH we
-  # minimally use -msse4.1. This seems to work with MKL.
+  # only by used with BUILD_ARCH=native. 
+  include(FindSSE)
   set(INTRINSICS "")
   list(APPEND INTRINSICS_NVCC)
 
   if(BUILD_ARCH STREQUAL "native")
     message(STATUS "Checking support for CPU intrinsics")
-    include(FindSSE)
     if(SSE2_FOUND)
       message(STATUS "SSE2 support found")
       set(INTRINSICS "${INTRINSICS} -msse2")
@@ -249,14 +262,28 @@ else(MSVC)
     # -msse4.1 once marian can solely be compiled with intgemm ("onnxjs" will be removed in that case)
     set(INTRINSICS "-mssse3 -msimd128")
   else()
-    set(INTRINSICS "-msse4.1")
+     # For overridden BUILD_ARCH we minimally use -msse4.1 (if available)
+    if(SSE4_1_FOUND)
+      message(STATUS "SSE4.1 support found")
+      set(INTRINSICS "${INTRINSICS} -msse4.1")
+      list(APPEND INTRINSICS_NVCC -Xcompiler\ -msse4.1)
+    endif(SSE4_1_FOUND)
   endif()
 
   if(USE_FBGEMM)
     set(EXT_LIBS ${EXT_LIBS} fbgemm dl)
     add_definitions(-DUSE_FBGEMM=1)
   endif(USE_FBGEMM)
 
+  if(USE_INTGEMM)
+    add_definitions(-DUSE_INTGEMM=1)
+  endif(USE_INTGEMM)
+
+  if(USE_SIMDE)
+    add_definitions(-DUSE_SIMDE=1)
+    add_definitions(-DSIMDE_ENABLE_NATIVE_ALIASES=1)
+  endif(USE_SIMDE)
+
   if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0)
     # Clang-10.0.0 complains when CUDA is newer than 10.1
     set(CLANG_IGNORE_UNKNOWN_CUDA "-Wno-unknown-warning-option -Wno-unknown-cuda-version")
@@ -266,7 +293,8 @@ else(MSVC)
   set(DISABLE_GLOBALLY "-Wno-unused-result ${CLANG_IGNORE_UNKNOWN_CUDA} ${CLANG_IGNORE_UNUSED_VALUES}") # This needs to appear here as well to appease clang11+ on linux
 
   # These are used in src/CMakeLists.txt on a per-target basis
-  list(APPEND ALL_WARNINGS -Wall; -Werror; -Wextra; -Wno-unused-result; -Wno-deprecated;
+  list(APPEND ALL_WARNINGS -Wall; # -Werror; 
+    -Wextra; -Wno-unused-result; -Wno-deprecated;
     -Wno-pragmas; -Wno-unused-parameter; -Wno-unused-function;
     -Wno-unused-value; -Wno-unknown-pragmas; -Wno-sign-compare;
     -Wno-missing-field-initializers; ${CLANG_IGNORE_UNUSED_PRIVATE_FIELD})
@@ -542,24 +570,32 @@ endif(USE_MPI)
 ###############################################################################
 # Find BLAS library for CPU compilation
 if(COMPILE_CPU)
-  set(EXT_LIBS ${EXT_LIBS} intgemm)       # Move the intgemm bits on top since they compile with every single variant
+  if(USE_INTGEMM)
+      set(EXT_LIBS ${EXT_LIBS} intgemm)       # Move the intgemm bits on top since they compile with every single variant
+  endif(USE_INTGEMM)
+
+  if(USE_RUY)
+    set(EXT_LIBS ${EXT_LIBS} ruy)
+  endif(USE_RUY)
+
   add_definitions(-DCOMPILE_CPU=1)        # Move the compile CPU definition on top since we want to compile intgemm when we set compile CPU
                                           # in case a BLAS vendor is not found, we have a runtime error, although we should probably not allow the compilation to go on
                                           # if there are BLAS vendors, we have other runtime checks with sane error messages.
-  if(USE_WASM_COMPATIBLE_BLAS)
+  if(USE_ONNX_SGEMM)
     ## Use a wasm compatible BLAS
+    ## ^ SGEMM != BLAS
     set(EXT_LIBS ${EXT_LIBS} onnx-sgemm)
-    set(BLAS_FOUND TRUE)
-    set(BLAS_VENDOR "ONNX-SGEMM")
-    add_definitions(-DBLAS_FOUND=1  -DWASM_COMPATIBLE_BLAS=1) # Might be required in some cmake files further down the line, let's avoid using add_compile_definitions in this codeblock
-  elseif(APPLE AND USE_APPLE_ACCELERATE)
+    add_definitions(-DUSE_ONNX_SGEMM=1) # Might be required in some cmake files further down the line, let's avoid using add_compile_definitions in this codeblock
+  endif(USE_ONNX_SGEMM)
+
+  if(APPLE AND USE_APPLE_ACCELERATE)
     set(BLAS_VENDOR "Accelerate")
     # see https://developer.apple.com/documentation/accelerate for more info
     # you may need to install Xcode command line tools if you don't have them already (https://developer.apple.com/xcode/features/)
     include_directories("/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/Headers")
     set(EXT_LIBS ${EXT_LIBS} "-framework Accelerate")
     add_definitions(-DBLAS_FOUND=1)
-  else(USE_WASM_COMPATIBLE_BLAS)
+  else(USE_ONNX_SGEMM)
     if(USE_MKL)
       find_package(MKL)
     endif(USE_MKL)
@@ -580,7 +616,7 @@ if(COMPILE_CPU)
         endif(CBLAS_FOUND)
       endif(BLAS_FOUND)
     endif(MKL_FOUND)
-  endif(USE_WASM_COMPATIBLE_BLAS)
+  endif(USE_ONNX_SGEMM)
 endif(COMPILE_CPU)
 
 ###############################################################################

scripts/run.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+NDK=/mnt/Storage/jphilip/android-ndk-r23b
+ABI=arm64-v8a
+MINSDK_VERSION=28
+CUSTOM_MODULE_PATH=/mnt/Storage/jphilip/marian-android/openblas-install/lib/cmake/openblas
+ANDROID_PLATFORM=28
+
+OTHER_ANDROID_ARGS=(
+    -DANDROID_ARM_NEON=TRUE
+)
+
+OTHER_MARIAN_ARGS=(
+    -DCOMPILE_CUDA=off
+    -DCOMPILE_CPU=on
+    -DCMAKE_HAVE_THREADS_LIBRARY=1
+    -DCMAKE_USE_WIN32_THREADS_INIT=0
+    -DCMAKE_USE_PTHREADS_INIT=1
+    -DTHREADS_PREFER_PTHREAD_FLAG=ON
+    -DBUILD_ARCH=armv8-a
+    -DUSE_INTGEMM=off
+    -DUSE_SIMDE=on
+    -DUSE_RUY=on
+    -DUSE_ONNX_SGEMM=on # For time being.
+    -DCOMPILE_WITHOUT_EXCEPTIONS=on # Apparently this can reduce the binary size, let's see.
+)
+# Additionally list variables finally configured.
+cmake -L \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \
+    -DCMAKE_MODULE_PATH=$CUSTOM_MODULE_PATH \
+    -DANDROID_TOOLCHAIN=clang \
+    -DANDROID_ABI=$ABI \
+    -DANDROID_PLATFORM=$ANDROID_PLATFORM \
+    -DANDROID_NATIVE_API_LEVEL=$MINSDKVERSION \
+    -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi-4.8 \
+    -DANDROID_STL=c++_static \
+    -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+    "${OTHER_ANDROID_ARGS[@]}" "${OTHER_MARIAN_ARGS[@]}" \
+    ..