Merge pull request #1016 from AntelopeIO/oc_opt_small_const_memcpy

workaround cdt's small const memcpy host function calls in EOS VM OC

libraries/chain/include/eosio/chain/unapplied_transaction_queue.hpp

@@ -8,12 +8,6 @@
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/member.hpp>
 
-namespace fc {
-  inline std::size_t hash_value( const fc::sha256& v ) {
-     return v._hash[3];
-  }
-}
-
 namespace eosio { namespace chain {
 
 using namespace boost::multi_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/code_cache.hpp

@@ -17,12 +17,6 @@
 
 #include <thread>
 
-namespace fc {
-   inline size_t hash_value(const fc::sha256& code_id) {
-      return boost::hash<fc::sha256>()(code_id);
-   }
-}
-
 namespace eosio { namespace chain { namespace eosvmoc {
 
 using namespace boost::multi_index;

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/eos-vm-oc.hpp

@@ -44,7 +44,7 @@ enum eosvmoc_exitcode : int {
    EOSVMOC_EXIT_EXCEPTION
 };
 
-static constexpr uint8_t current_codegen_version = 1;
+static constexpr uint8_t current_codegen_version = 2;
 
 }
 

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic.hpp

@@ -25,4 +25,7 @@ using intrinsic_map_t = std::map<std::string, intrinsic_entry>;
 
 const intrinsic_map_t& get_intrinsic_map();
 
+static constexpr unsigned minimum_const_memcpy_intrinsic_to_optimize = 1;
+static constexpr unsigned maximum_const_memcpy_intrinsic_to_optimize = 128;
+
 }}}

libraries/chain/include/eosio/chain/webassembly/eos-vm-oc/intrinsic_mapping.hpp

@@ -278,7 +278,8 @@ inline constexpr auto get_intrinsic_table() {
       "env.bls_fp_mod",
       "env.bls_fp_mul",
       "env.bls_fp_exp",
-      "env.set_finalizers"
+      "env.set_finalizers",
+      "eosvmoc_internal.check_memcpy_params"
    );
 }
 inline constexpr std::size_t find_intrinsic_index(std::string_view hf) {

libraries/chain/webassembly/runtimes/eos-vm-oc/LLVMEmitIR.cpp

@@ -697,12 +697,14 @@ namespace LLVMJIT
 			llvm::Value* callee;
 			const FunctionType* calleeType;
 			bool isExit = false;
+			bool isMemcpy = false;
 			if(imm.functionIndex < moduleContext.importedFunctionOffsets.size())
 			{
 				calleeType = module.types[module.functions.imports[imm.functionIndex].type.index];
 				llvm::Value* ic = irBuilder.CreateLoad( emitLiteralPointer((void*)(OFFSET_OF_FIRST_INTRINSIC-moduleContext.importedFunctionOffsets[imm.functionIndex]*8), llvmI64Type->getPointerTo(256)) );
 				callee = irBuilder.CreateIntToPtr(ic, asLLVMType(calleeType)->getPointerTo());
 				isExit = module.functions.imports[imm.functionIndex].moduleName == "env" && module.functions.imports[imm.functionIndex].exportName == "eosio_exit";
+				isMemcpy = module.functions.imports[imm.functionIndex].moduleName == "env" && module.functions.imports[imm.functionIndex].exportName == "memcpy";
 			}
 			else
 			{
@@ -715,6 +717,28 @@ namespace LLVMJIT
 			auto llvmArgs = (llvm::Value**)alloca(sizeof(llvm::Value*) * calleeType->parameters.size());
 			popMultiple(llvmArgs,calleeType->parameters.size());
 
+			//convert small constant sized memcpy host function calls to a load+store (plus small call to validate non-overlap rule)
+			if(isMemcpy) {
+				assert(calleeType->parameters.size() == 3);
+				if(llvm::ConstantInt* const_memcpy_sz = llvm::dyn_cast<llvm::ConstantInt>(llvmArgs[2]);
+				     const_memcpy_sz &&
+					  const_memcpy_sz->getZExtValue() >= minimum_const_memcpy_intrinsic_to_optimize &&
+					  const_memcpy_sz->getZExtValue() <= maximum_const_memcpy_intrinsic_to_optimize) {
+					const unsigned sz_value = const_memcpy_sz->getZExtValue();
+					llvm::IntegerType* type_of_memcpy_width = llvm::Type::getIntNTy(context, sz_value*8);
+
+					llvm::Value* load_pointer = coerceByteIndexToPointer(llvmArgs[1],0,type_of_memcpy_width);
+					llvm::Value* store_pointer = coerceByteIndexToPointer(llvmArgs[0],0,type_of_memcpy_width);
+					irBuilder.CreateStore(irBuilder.CreateLoad(load_pointer), store_pointer, true);
+
+					emitRuntimeIntrinsic("eosvmoc_internal.check_memcpy_params",
+					                     FunctionType::get(ResultType::none,{ValueType::i32,ValueType::i32,ValueType::i32}),
+						                  {llvmArgs[0],llvmArgs[1],llvmArgs[2]});
+					push(llvmArgs[0]);
+					return;
+				}
+			}
+
 			// Call the function.
 			auto result = createCall(callee,llvm::ArrayRef<llvm::Value*>(llvmArgs,calleeType->parameters.size()));
 			if(isExit) {

libraries/chain/webassembly/runtimes/eos-vm-oc/executor.cpp

@@ -87,8 +87,9 @@ static intrinsic eosio_exit_intrinsic("env.eosio_exit", IR::FunctionType::get(IR
   std::integral_constant<std::size_t, find_intrinsic_index("env.eosio_exit")>::value
 );
 
-static void throw_internal_exception(const char* const s) {
-   *reinterpret_cast<std::exception_ptr*>(eos_vm_oc_get_exception_ptr()) = std::make_exception_ptr(wasm_execution_error(FC_LOG_MESSAGE(error, s)));
+template <typename E>
+static void throw_internal_exception(const E& e) {
+   *reinterpret_cast<std::exception_ptr*>(eos_vm_oc_get_exception_ptr()) = std::make_exception_ptr(e);
    siglongjmp(*eos_vm_oc_get_jmp_buf(), EOSVMOC_EXIT_EXCEPTION);
    __builtin_unreachable();
 }
@@ -101,25 +102,42 @@ static void throw_internal_exception(const char* const s) {
 	void name()
 
 DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,depth_assert) {
-   throw_internal_exception("Exceeded call depth maximum");
+   throw_internal_exception(wasm_execution_error(FC_LOG_MESSAGE(error, "Exceeded call depth maximum")));
 }
 
 DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,div0_or_overflow) {
-   throw_internal_exception("Division by 0 or integer overflow trapped");
+   throw_internal_exception(wasm_execution_error(FC_LOG_MESSAGE(error, "Division by 0 or integer overflow trapped")));
 }
 
 DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,indirect_call_mismatch) {
-   throw_internal_exception("Indirect call function type mismatch");
+   throw_internal_exception(wasm_execution_error(FC_LOG_MESSAGE(error, "Indirect call function type mismatch")));
 }
 
 DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,indirect_call_oob) {
-   throw_internal_exception("Indirect call index out of bounds");
+   throw_internal_exception(wasm_execution_error(FC_LOG_MESSAGE(error, "Indirect call index out of bounds")));
 }
 
 DEFINE_EOSVMOC_TRAP_INTRINSIC(eosvmoc_internal,unreachable) {
-   throw_internal_exception("Unreachable reached");
+   throw_internal_exception(wasm_execution_error(FC_LOG_MESSAGE(error, "Unreachable reached")));
 }
 
+static void eos_vm_oc_check_memcpy_params(int32_t dest, int32_t src, int32_t length) {
+   //make sure dest & src are zexted when converted from signed 32-bit to signed ptrdiff_t; length should always be small but do it too
+   const unsigned udest = dest;
+   const unsigned usrc = src;
+   const unsigned ulength = length;
+
+   //this must remain the same behavior as the memcpy host function
+   if((size_t)(std::abs((ptrdiff_t)udest - (ptrdiff_t)usrc)) >= ulength)
+      return;
+   throw_internal_exception(overlapping_memory_error(FC_LOG_MESSAGE(error, "memcpy can only accept non-aliasing pointers")));
+}
+
+static intrinsic check_memcpy_params_intrinsic("eosvmoc_internal.check_memcpy_params", IR::FunctionType::get(IR::ResultType::none,{IR::ValueType::i32,IR::ValueType::i32,IR::ValueType::i32}),
+  (void*)&eos_vm_oc_check_memcpy_params,
+  std::integral_constant<std::size_t, find_intrinsic_index("eosvmoc_internal.check_memcpy_params")>::value
+);
+
 struct executor_signal_init {
    executor_signal_init() {
       struct sigaction sig_action, old_sig_action;

libraries/libfc/include/fc/crypto/sha256.hpp

@@ -147,5 +147,12 @@ namespace boost
        }
     };
 }
+
+namespace fc {
+   inline size_t hash_value(const fc::sha256& s) {
+      return boost::hash<fc::sha256>()(s);
+   }
+}
+
 #include <fc/reflect/reflect.hpp>
 FC_REFLECT_TYPENAME( fc::sha256 )

unittests/api_tests.cpp

@@ -16,6 +16,7 @@
 #include <eosio/chain/wasm_interface.hpp>
 #include <eosio/chain/resource_limits.hpp>
 #include <eosio/chain/finalizer_authority.hpp>
+#include <eosio/chain/webassembly/eos-vm-oc.hpp>
 
 #include <fc/crypto/digest.hpp>
 #include <fc/crypto/sha256.hpp>
@@ -3352,6 +3353,86 @@ BOOST_AUTO_TEST_CASE_TEMPLATE( get_code_hash_tests, T, validating_testers ) { tr
    check("test"_n, 3);
 } FC_LOG_AND_RETHROW() }
 
+BOOST_AUTO_TEST_CASE_TEMPLATE( small_const_memcpy_tests, T, validating_testers ) { try {
+   T t;
+   t.create_account("smallmemcpy"_n);
+   t.produce_block();
+
+   for(unsigned i = eosvmoc::minimum_const_memcpy_intrinsic_to_optimize; i <= eosvmoc::maximum_const_memcpy_intrinsic_to_optimize; ++i) {
+      t.set_code("smallmemcpy"_n, fc::format_string(small_memcpy_const_dstsrc_wastfmt, fc::mutable_variant_object("COPY_SIZE", i)).c_str());
+
+      signed_transaction trx;
+      action act;
+      act.account = "smallmemcpy"_n;
+      act.name = ""_n;
+      act.authorization = vector<permission_level>{{"smallmemcpy"_n,config::active_name}};
+      act.data.push_back(i);
+      trx.actions.push_back(act);
+      t.set_transaction_headers(trx);
+      trx.sign(t.get_private_key( "smallmemcpy"_n, "active" ), t.get_chain_id());
+      t.push_transaction(trx);
+
+      if(i%10 == 0)
+         t.produce_block();
+   }
+
+} FC_LOG_AND_RETHROW() }
+
+//similar to above, but the source and destination values passed to memcpy are not consts
+BOOST_AUTO_TEST_CASE_TEMPLATE( small_var_memcpy_tests, T, validating_testers ) { try {
+   T t;
+   t.create_account("smallmemcpy"_n);
+   t.produce_block();
+
+   for(unsigned i = eosvmoc::minimum_const_memcpy_intrinsic_to_optimize; i <= eosvmoc::maximum_const_memcpy_intrinsic_to_optimize; ++i) {
+      t.set_code("smallmemcpy"_n, fc::format_string(small_memcpy_var_dstsrc_wastfmt, fc::mutable_variant_object("COPY_SIZE", i)).c_str());
+
+      signed_transaction trx;
+      action act;
+      act.account = "smallmemcpy"_n;
+      act.name = ""_n;
+      act.authorization = vector<permission_level>{{"smallmemcpy"_n,config::active_name}};
+      act.data.push_back(i);
+      trx.actions.push_back(act);
+      t.set_transaction_headers(trx);
+      trx.sign(t.get_private_key( "smallmemcpy"_n, "active" ), t.get_chain_id());
+      t.push_transaction(trx);
+
+      if(i%10 == 0)
+         t.produce_block();
+   }
+
+} FC_LOG_AND_RETHROW() }
+
+//check that small constant sized memcpys (that OC will optimize "away") correctly fail on edge or high side of invalid memory
+BOOST_AUTO_TEST_CASE_TEMPLATE( small_const_memcpy_oob_tests, T, validating_testers ) { try {
+   T t;
+   t.create_account("smallmemcpy"_n);
+   t.produce_block();
+
+   auto sendit = [&]() {
+      signed_transaction trx;
+      action act;
+      act.account = "smallmemcpy"_n;
+      act.name = ""_n;
+      act.authorization = vector<permission_level>{{"smallmemcpy"_n,config::active_name}};
+      trx.actions.push_back(act);
+      t.set_transaction_headers(trx);
+      trx.sign(t.get_private_key( "smallmemcpy"_n, "active" ), t.get_chain_id());
+      t.push_transaction(trx);
+   };
+
+   t.set_code("smallmemcpy"_n, small_memcpy_overlapenddst_wast);
+   BOOST_REQUIRE_EXCEPTION(sendit(), eosio::chain::wasm_execution_error, [](const eosio::chain::wasm_execution_error& e) {return expect_assert_message(e, "access violation");});
+
+   t.set_code("smallmemcpy"_n, small_memcpy_overlapendsrc_wast);
+   BOOST_REQUIRE_EXCEPTION(sendit(), eosio::chain::wasm_execution_error, [](const eosio::chain::wasm_execution_error& e) {return expect_assert_message(e, "access violation");});
+
+   t.set_code("smallmemcpy"_n, small_memcpy_high_wast);
+   BOOST_REQUIRE_EXCEPTION(sendit(), eosio::chain::wasm_execution_error, [](const eosio::chain::wasm_execution_error& e) {return expect_assert_message(e, "access violation");});
+
+} FC_LOG_AND_RETHROW() }
+
 //test that find_secondary_key behaves like lowerbound
 BOOST_AUTO_TEST_CASE_TEMPLATE( find_seconary_key, T, validating_testers ) {
    try {

unittests/contracts/test_wasts.hpp

@@ -1037,4 +1037,84 @@ static const char divmod_host_function_overflow_wast[] = R"=====(
       ))
    )
 )
+)=====";
+
+static const char small_memcpy_const_dstsrc_wastfmt[] = R"=====(
+(module
+   (import "env" "memcpy" (func $$memcpy (param i32 i32 i32) (result i32)))
+   (import "env" "memcmp" (func $$memcmp (param i32 i32 i32) (result i32)))
+   (export "apply" (func $$apply))
+   (memory 1)
+   (func $$apply (param i64) (param i64) (param i64)
+      ;; do copy and check that return value is dst
+      (if (i32.ne (call $$memcpy (i32.const 256) (i32.const 64) (i32.const ${COPY_SIZE})) (i32.const 256)) (then unreachable))
+
+      ;; validate copied region
+      (if (i32.ne (call $$memcmp (i32.const 256) (i32.const 64) (i32.const ${COPY_SIZE})) (i32.const 0)) (then unreachable))
+
+      ;; check the 4 bytes before and and after the copied region and expect them to still be 0x0
+      (if (i32.ne (i32.load (i32.const 252)) (i32.const 0)) (then unreachable))
+      (if (i32.ne (i32.load (i32.add (i32.const 256) (i32.const ${COPY_SIZE}))) (i32.const 0)) (then unreachable))
+   )
+   (data (i32.const 64) "1234567890-abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+)
+)=====";
+
+static const char small_memcpy_var_dstsrc_wastfmt[] = R"=====(
+(module
+   (import "env" "memcpy" (func $$memcpy (param i32 i32 i32) (result i32)))
+   (import "env" "memcmp" (func $$memcmp (param i32 i32 i32) (result i32)))
+   (export "apply" (func $$apply))
+   (memory 1)
+   (func $$doit (param $$dst i32) (param $$src i32)
+      ;; do copy and check that return value is dst
+      (if (i32.ne (call $$memcpy (get_local $$dst) (get_local $$src) (i32.const ${COPY_SIZE})) (get_local $$dst)) (then unreachable))
+
+      ;; validate copied region
+      (if (i32.ne (call $$memcmp (get_local $$dst) (get_local $$src) (i32.const ${COPY_SIZE})) (i32.const 0)) (then unreachable))
+
+      ;; check the 4 bytes before and and after the copied region and expect them to still be 0x0
+      (if (i32.ne (i32.load (i32.sub (get_local $$dst) (i32.const 4))) (i32.const 0)) (then unreachable))
+      (if (i32.ne (i32.load (i32.add (get_local $$dst) (i32.const ${COPY_SIZE}))) (i32.const 0)) (then unreachable))
+   )
+   (func $$apply (param i64) (param i64) (param i64)
+      (call $$doit (i32.const 256) (i32.const 64))
+   )
+   (data (i32.const 64) "1234567890-abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+)
+)=====";
+
+static const char small_memcpy_overlapenddst_wast[] = R"=====(
+(module
+   (import "env" "memcpy" (func $memcpy (param i32 i32 i32) (result i32)))
+   (export "apply" (func $apply))
+   (memory 1)
+   (func $apply (param i64) (param i64) (param i64)
+      (drop (call $memcpy (i32.const 65532) (i32.const 64) (i32.const 8)))
+   )
+
+   (data (i32.const 64) "1234567890-abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+)
+)=====";
+
+static const char small_memcpy_overlapendsrc_wast[] = R"=====(
+(module
+   (import "env" "memcpy" (func $memcpy (param i32 i32 i32) (result i32)))
+   (export "apply" (func $apply))
+   (memory 1)
+   (func $apply (param i64) (param i64) (param i64)
+      (drop (call $memcpy (i32.const 4) (i32.const 65532) (i32.const 8)))
+   )
+)
+)=====";
+
+static const char small_memcpy_high_wast[] = R"=====(
+(module
+   (import "env" "memcpy" (func $memcpy (param i32 i32 i32) (result i32)))
+   (export "apply" (func $apply))
+   (memory 1)
+   (func $apply (param i64) (param i64) (param i64)
+      (drop (call $memcpy (i32.const 4294967295) (i32.const 4294967200) (i32.const 8)))
+   )
+)
 )=====";