perf(verification): add multithreading to tx scripts verification

hathor/transaction/scripts/__init__.py

@@ -18,7 +18,7 @@
     get_sigops_count,
     parse_address_script,
 )
-from hathor.transaction.scripts.execute import ScriptExtras, script_eval
+from hathor.transaction.scripts.execute import ScriptExtras, evaluate_scripts
 from hathor.transaction.scripts.hathor_script import HathorScript
 from hathor.transaction.scripts.multi_sig import MultiSig
 from hathor.transaction.scripts.nano_contract_match_values import NanoContractMatchValues
@@ -35,6 +35,6 @@
     'parse_address_script',
     'create_base_script',
     'create_output_script',
-    'script_eval',
+    'evaluate_scripts',
     'get_sigops_count',
 ]

hathor/transaction/scripts/execute.py

@@ -13,6 +13,7 @@
 #  limitations under the License.
 
 import struct
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 from typing import NamedTuple, Optional, Union
 
 from hathor.transaction import BaseTransaction, Transaction, TxInput
@@ -88,44 +89,59 @@ def evaluate_final_stack(stack: Stack, log: list[str]) -> None:
         raise FinalStackInvalid('\n'.join(log))
 
 
-def script_eval(tx: Transaction, txin: TxInput, spent_tx: BaseTransaction) -> None:
+def evaluate_scripts(tx: Transaction) -> None:
     """Evaluates the output script and input data according to
-    a very limited subset of Bitcoin's scripting language.
+    a very limited subset of Bitcoin's scripting language, for all inputs in the tx.
 
     :param tx: the transaction being validated, the 'owner' of the input data
     :type tx: :py:class:`hathor.transaction.Transaction`
 
-    :param txin: transaction input being evaluated
-    :type txin: :py:class:`hathor.transaction.TxInput`
-
-    :param spent_tx: the transaction referenced by the input
-    :type spent_tx: :py:class:`hathor.transaction.BaseTransaction`
-
     :raises ScriptError: if script verification fails
     """
-    input_data = txin.data
-    output_script = spent_tx.outputs[txin.index].script
-    log: list[str] = []
-    extras = ScriptExtras(tx=tx, txin=txin, spent_tx=spent_tx)
-
     from hathor.transaction.scripts import MultiSig
-    if MultiSig.re_match.search(output_script):
-        # For MultiSig there are 2 executions:
-        # First we need to evaluate that redeem_script matches redeem_script_hash
-        # we can't use input_data + output_script because it will end with an invalid stack
-        # i.e. the signatures will still be on the stack after ouput_script is executed
-        redeem_script_pos = MultiSig.get_multisig_redeem_script_pos(input_data)
-        full_data = txin.data[redeem_script_pos:] + output_script
-        execute_eval(full_data, log, extras)
-
-        # Second, we need to validate that the signatures on the input_data solves the redeem_script
-        # we pop and append the redeem_script to the input_data and execute it
-        multisig_data = MultiSig.get_multisig_data(extras.txin.data)
-        execute_eval(multisig_data, log, extras)
-    else:
-        # merge input_data and output_script
-        full_data = input_data + output_script
-        execute_eval(full_data, log, extras)
+    with ThreadPoolExecutor() as executor:
+        futures: list[Future[None]] = []
+        for tx_input in tx.inputs:
+            spent_tx = tx.get_spent_tx(tx_input)
+            output_script = spent_tx.outputs[tx_input.index].script
+            extras = ScriptExtras(tx=tx, txin=tx_input, spent_tx=spent_tx)
+
+            if MultiSig.re_match.search(output_script):
+                future = executor.submit(
+                    _evaluate_multisig,
+                    input_data=tx_input.data,
+                    output_script=output_script,
+                    extras=extras,
+                )
+            else:
+                future = executor.submit(
+                    execute_eval,
+                    data=tx_input.data + output_script,
+                    log=[],
+                    extras=extras,
+                )
+
+            futures.append(future)
+
+        for future in as_completed(futures):
+            future.result()
+
+
+def _evaluate_multisig(*, input_data: bytes, output_script: bytes, extras: ScriptExtras) -> None:
+    from hathor.transaction.scripts import MultiSig
+    log: list[str] = []
+    # For MultiSig there are 2 executions:
+    # First we need to evaluate that redeem_script matches redeem_script_hash
+    # we can't use input_data + output_script because it will end with an invalid stack
+    # i.e. the signatures will still be on the stack after output_script is executed
+    redeem_script_pos = MultiSig.get_multisig_redeem_script_pos(input_data)
+    full_data = input_data[redeem_script_pos:] + output_script
+    execute_eval(full_data, log, extras)
+
+    # Second, we need to validate that the signatures on the input_data solves the redeem_script
+    # we pop and append the redeem_script to the input_data and execute it
+    multisig_data = MultiSig.get_multisig_data(input_data)
+    execute_eval(multisig_data, log, extras)
 
 
 def decode_opn(opcode: int) -> int:

hathor/verification/transaction_verifier.py

@@ -17,7 +17,7 @@
 from hathor.profiler import get_cpu_profiler
 from hathor.reward_lock import get_spent_reward_locked_info
 from hathor.reward_lock.reward_lock import get_minimum_best_height
-from hathor.transaction import BaseTransaction, Transaction, TxInput
+from hathor.transaction import Transaction
 from hathor.transaction.exceptions import (
     ConflictingInputs,
     DuplicatedParents,
@@ -120,25 +120,23 @@ def verify_inputs(self, tx: Transaction, *, skip_script: bool = False) -> None:
                     spent_tx.timestamp,
                 ))
 
-            if not skip_script:
-                self.verify_script(tx=tx, input_tx=input_tx, spent_tx=spent_tx)
-
             # check if any other input in this tx is spending the same output
             key = (input_tx.tx_id, input_tx.index)
             if key in spent_outputs:
                 raise ConflictingInputs('tx {} inputs spend the same output: {} index {}'.format(
                     tx.hash_hex, input_tx.tx_id.hex(), input_tx.index))
             spent_outputs.add(key)
 
-    def verify_script(self, *, tx: Transaction, input_tx: TxInput, spent_tx: BaseTransaction) -> None:
+        if not skip_script:
+            self.verify_scripts(tx)
+
+    def verify_scripts(self, tx: Transaction) -> None:
         """
         :type tx: Transaction
-        :type input_tx: TxInput
-        :type spent_tx: Transaction
         """
-        from hathor.transaction.scripts import script_eval
+        from hathor.transaction.scripts import evaluate_scripts
         try:
-            script_eval(tx, input_tx, spent_tx)
+            evaluate_scripts(tx)
         except ScriptError as e:
             raise InvalidInputData(e) from e
 

tests/tx/test_multisig.py

@@ -4,7 +4,7 @@
 from hathor.simulator.utils import add_new_blocks
 from hathor.transaction import Transaction, TxInput, TxOutput
 from hathor.transaction.exceptions import ScriptError
-from hathor.transaction.scripts import P2PKH, MultiSig, create_output_script, parse_address_script, script_eval
+from hathor.transaction.scripts import P2PKH, MultiSig, create_output_script, evaluate_scripts, parse_address_script
 from hathor.wallet.base_wallet import WalletBalance, WalletOutputInfo
 from hathor.wallet.util import generate_multisig_address, generate_multisig_redeem_script, generate_signature
 from tests import unittest
@@ -134,7 +134,7 @@ def test_spend_multisig(self):
         expected_dict = {'type': 'MultiSig', 'address': self.multisig_address_b58, 'timelock': None}
         self.assertEqual(cls_script.to_human_readable(), expected_dict)
 
-        script_eval(tx, tx_input, tx1)
+        evaluate_scripts(tx)
 
         # Script error
         with self.assertRaises(ScriptError):

tests/tx/test_nano_contracts.py

@@ -1,9 +1,10 @@
 import base64
+from unittest.mock import Mock
 
 import base58
 
 from hathor.transaction import Transaction, TxInput, TxOutput
-from hathor.transaction.scripts import P2PKH, NanoContractMatchValues, script_eval
+from hathor.transaction.scripts import P2PKH, NanoContractMatchValues, evaluate_scripts
 from hathor.util import json_dumpb
 from tests import unittest
 
@@ -37,5 +38,8 @@ def test_match_values(self):
             base64.b64decode(oracle_data), base64.b64decode(oracle_signature), base64.b64decode(pubkey))
         txin = TxInput(b'aa', 0, input_data)
         spent_tx = Transaction(outputs=[TxOutput(20, script)])
-        tx = Transaction(outputs=[TxOutput(20, P2PKH.create_output_script(address))])
-        script_eval(tx, txin, spent_tx)
+        tx_mock = Mock()
+        tx_mock.outputs = [TxOutput(20, P2PKH.create_output_script(address))]
+        tx_mock.inputs = [txin]
+        tx_mock.get_spent_tx = Mock(return_value=spent_tx)
+        evaluate_scripts(tx_mock)

tests/tx/test_verification.py

@@ -601,7 +601,7 @@ def test_transaction_verify(self) -> None:
         verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output)
         verify_sigops_input_wrapped = Mock(wraps=self.verifiers.tx.verify_sigops_input)
         verify_inputs_wrapped = Mock(wraps=self.verifiers.tx.verify_inputs)
-        verify_script_wrapped = Mock(wraps=self.verifiers.tx.verify_script)
+        verify_scripts_wrapped = Mock(wraps=self.verifiers.tx.verify_scripts)
         verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents)
         verify_sum_wrapped = Mock(wraps=self.verifiers.tx.verify_sum)
         verify_reward_locked_wrapped = Mock(wraps=self.verifiers.tx.verify_reward_locked)
@@ -615,7 +615,7 @@ def test_transaction_verify(self) -> None:
             patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped),
             patch.object(TransactionVerifier, 'verify_sigops_input', verify_sigops_input_wrapped),
             patch.object(TransactionVerifier, 'verify_inputs', verify_inputs_wrapped),
-            patch.object(TransactionVerifier, 'verify_script', verify_script_wrapped),
+            patch.object(TransactionVerifier, 'verify_scripts', verify_scripts_wrapped),
             patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped),
             patch.object(TransactionVerifier, 'verify_sum', verify_sum_wrapped),
             patch.object(TransactionVerifier, 'verify_reward_locked', verify_reward_locked_wrapped),
@@ -633,7 +633,7 @@ def test_transaction_verify(self) -> None:
         verify_sigops_output_wrapped.assert_called_once()
         verify_sigops_input_wrapped.assert_called_once()
         verify_inputs_wrapped.assert_called_once()
-        verify_script_wrapped.assert_called_once()
+        verify_scripts_wrapped.assert_called_once()
         verify_parents_wrapped.assert_called_once()
         verify_sum_wrapped.assert_called_once()
         verify_reward_locked_wrapped.assert_called_once()
@@ -735,7 +735,7 @@ def test_transaction_validate_full(self) -> None:
         verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output)
         verify_sigops_input_wrapped = Mock(wraps=self.verifiers.tx.verify_sigops_input)
         verify_inputs_wrapped = Mock(wraps=self.verifiers.tx.verify_inputs)
-        verify_script_wrapped = Mock(wraps=self.verifiers.tx.verify_script)
+        verify_scripts_wrapped = Mock(wraps=self.verifiers.tx.verify_scripts)
         verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents)
         verify_sum_wrapped = Mock(wraps=self.verifiers.tx.verify_sum)
         verify_reward_locked_wrapped = Mock(wraps=self.verifiers.tx.verify_reward_locked)
@@ -752,7 +752,7 @@ def test_transaction_validate_full(self) -> None:
             patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped),
             patch.object(TransactionVerifier, 'verify_sigops_input', verify_sigops_input_wrapped),
             patch.object(TransactionVerifier, 'verify_inputs', verify_inputs_wrapped),
-            patch.object(TransactionVerifier, 'verify_script', verify_script_wrapped),
+            patch.object(TransactionVerifier, 'verify_scripts', verify_scripts_wrapped),
             patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped),
             patch.object(TransactionVerifier, 'verify_sum', verify_sum_wrapped),
             patch.object(TransactionVerifier, 'verify_reward_locked', verify_reward_locked_wrapped),
@@ -773,7 +773,7 @@ def test_transaction_validate_full(self) -> None:
         assert verify_sigops_output_wrapped.call_count == 2
         verify_sigops_input_wrapped.assert_called_once()
         verify_inputs_wrapped.assert_called_once()
-        verify_script_wrapped.assert_called_once()
+        verify_scripts_wrapped.assert_called_once()
         verify_parents_wrapped.assert_called_once()
         verify_sum_wrapped.assert_called_once()
         verify_reward_locked_wrapped.assert_called_once()
@@ -896,7 +896,7 @@ def test_token_creation_transaction_verify(self) -> None:
         verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output)
         verify_sigops_input_wrapped = Mock(wraps=self.verifiers.tx.verify_sigops_input)
         verify_inputs_wrapped = Mock(wraps=self.verifiers.tx.verify_inputs)
-        verify_script_wrapped = Mock(wraps=self.verifiers.tx.verify_script)
+        verify_scripts_wrapped = Mock(wraps=self.verifiers.tx.verify_scripts)
         verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents)
         verify_sum_wrapped = Mock(wraps=self.verifiers.tx.verify_sum)
         verify_reward_locked_wrapped = Mock(wraps=self.verifiers.tx.verify_reward_locked)
@@ -913,7 +913,7 @@ def test_token_creation_transaction_verify(self) -> None:
             patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped),
             patch.object(TransactionVerifier, 'verify_sigops_input', verify_sigops_input_wrapped),
             patch.object(TransactionVerifier, 'verify_inputs', verify_inputs_wrapped),
-            patch.object(TransactionVerifier, 'verify_script', verify_script_wrapped),
+            patch.object(TransactionVerifier, 'verify_scripts', verify_scripts_wrapped),
             patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped),
             patch.object(TransactionVerifier, 'verify_sum', verify_sum_wrapped),
             patch.object(TransactionVerifier, 'verify_reward_locked', verify_reward_locked_wrapped),
@@ -933,7 +933,7 @@ def test_token_creation_transaction_verify(self) -> None:
         verify_sigops_output_wrapped.assert_called_once()
         verify_sigops_input_wrapped.assert_called_once()
         verify_inputs_wrapped.assert_called_once()
-        verify_script_wrapped.assert_called_once()
+        verify_scripts_wrapped.assert_called_once()
         verify_parents_wrapped.assert_called_once()
         verify_sum_wrapped.assert_called_once()
         verify_reward_locked_wrapped.assert_called_once()
@@ -1038,7 +1038,7 @@ def test_token_creation_transaction_validate_full(self) -> None:
         verify_sigops_output_wrapped = Mock(wraps=self.verifiers.vertex.verify_sigops_output)
         verify_sigops_input_wrapped = Mock(wraps=self.verifiers.tx.verify_sigops_input)
         verify_inputs_wrapped = Mock(wraps=self.verifiers.tx.verify_inputs)
-        verify_script_wrapped = Mock(wraps=self.verifiers.tx.verify_script)
+        verify_scripts_wrapped = Mock(wraps=self.verifiers.tx.verify_scripts)
         verify_parents_wrapped = Mock(wraps=self.verifiers.vertex.verify_parents)
         verify_sum_wrapped = Mock(wraps=self.verifiers.tx.verify_sum)
         verify_reward_locked_wrapped = Mock(wraps=self.verifiers.tx.verify_reward_locked)
@@ -1058,7 +1058,7 @@ def test_token_creation_transaction_validate_full(self) -> None:
             patch.object(VertexVerifier, 'verify_sigops_output', verify_sigops_output_wrapped),
             patch.object(TransactionVerifier, 'verify_sigops_input', verify_sigops_input_wrapped),
             patch.object(TransactionVerifier, 'verify_inputs', verify_inputs_wrapped),
-            patch.object(TransactionVerifier, 'verify_script', verify_script_wrapped),
+            patch.object(TransactionVerifier, 'verify_scripts', verify_scripts_wrapped),
             patch.object(VertexVerifier, 'verify_parents', verify_parents_wrapped),
             patch.object(TransactionVerifier, 'verify_sum', verify_sum_wrapped),
             patch.object(TransactionVerifier, 'verify_reward_locked', verify_reward_locked_wrapped),
@@ -1081,7 +1081,7 @@ def test_token_creation_transaction_validate_full(self) -> None:
         assert verify_sigops_output_wrapped.call_count == 2
         verify_sigops_input_wrapped.assert_called_once()
         verify_inputs_wrapped.assert_called_once()
-        verify_script_wrapped.assert_called_once()
+        verify_scripts_wrapped.assert_called_once()
         verify_parents_wrapped.assert_called_once()
         verify_sum_wrapped.assert_called_once()
         verify_reward_locked_wrapped.assert_called_once()

tests/wallet/test_wallet_hd.py

@@ -40,8 +40,9 @@ def test_transaction_and_balance(self):
         out = WalletOutputInfo(decode_address(new_address2), self.TOKENS, timelock=None)
         tx1 = self.wallet.prepare_transaction_compute_inputs(Transaction, [out], self.tx_storage)
         tx1.update_hash()
+        tx1.storage = self.tx_storage
         verifier = self.manager.verification_service.verifiers.tx
-        verifier.verify_script(tx=tx1, input_tx=tx1.inputs[0], spent_tx=block)
+        verifier.verify_scripts(tx=tx1)
         tx1.storage = self.tx_storage
         tx1.get_metadata().validation = ValidationState.FULL
         self.wallet.on_new_tx(tx1)
@@ -62,7 +63,7 @@ def test_transaction_and_balance(self):
         tx2.storage = self.tx_storage
         tx2.update_hash()
         tx2.storage = self.tx_storage
-        verifier.verify_script(tx=tx2, input_tx=tx2.inputs[0], spent_tx=tx1)
+        verifier.verify_scripts(tx=tx2)
         tx2.get_metadata().validation = ValidationState.FULL
         tx2.init_static_metadata_from_storage(self._settings, self.tx_storage)
         self.tx_storage.save_transaction(tx2)