disable all the tvm tests in tests/

tests/test_backends.py

@@ -32,6 +32,7 @@
     sparkml_installed,
     pandas_installed,
     prophet_installed,
+    is_on_github_actions,
 )
 from hummingbird.ml.exceptions import MissingBackend
 
@@ -396,6 +397,10 @@ def test_load_fails_bad_path_onnx(self):
         self.assertRaises(AssertionError, hummingbird.ml.ONNXContainer.load, "nonsense")
 
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_load_fails_bad_path_tvm(self):
         self.assertRaises(AssertionError, hummingbird.ml.TVMContainer.load, "nonsense.zip")
         self.assertRaises(AssertionError, hummingbird.ml.TVMContainer.load, "nonsense")
@@ -434,6 +439,10 @@ def test_torchscript_test_data(self):
 
     # Test TVM requires test_data
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_test_data(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -451,6 +460,10 @@ def test_tvm_test_data(self):
 
     # Test tvm save and load
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_load(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -474,6 +487,10 @@ def test_tvm_save_load(self):
 
     # Test tvm save and load
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_load_digest(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -500,6 +517,10 @@ def test_tvm_save_load_digest(self):
 
     # Test tvm save and generic load
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_generic_load(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -523,6 +544,10 @@ def test_tvm_save_generic_load(self):
 
     # Test tvm save and load zip file
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_load_zip(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -545,6 +570,10 @@ def test_tvm_save_load_zip(self):
         os.remove("tvm-tmp.zip")
 
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_load_load(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -567,6 +596,10 @@ def test_tvm_save_load_load(self):
         os.remove("tvm-tmp.zip")
 
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_save_load_no_versions(self):
         from hummingbird.ml.operator_converters import constants
 

tests/test_extra_conf.py

@@ -22,6 +22,7 @@
     pandas_installed,
     lightgbm_installed,
     tvm_installed,
+    is_on_github_actions,
 )
 from hummingbird.ml import constants
 
@@ -423,6 +424,10 @@ def test_onnx_iforest_batch(self):
 
     # Test tvm transform with batching.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_batch_transform(self):
         warnings.filterwarnings("ignore")
         model = StandardScaler(with_mean=True, with_std=True)
@@ -440,6 +445,10 @@ def test_tvm_batch_transform(self):
 
     # Test tvm regression with batching.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_regression_batch(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -461,6 +470,10 @@ def test_tvm_regression_batch(self):
 
     # Test tvm classification with batching.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_classification_batch(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -482,6 +495,10 @@ def test_tvm_classification_batch(self):
 
     # Test tvm iforest with batching.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_iforest_batch(self):
         warnings.filterwarnings("ignore")
         num_classes = 2
@@ -503,6 +520,10 @@ def test_tvm_iforest_batch(self):
 
     # Test tvm transform with batching and uneven numer of records.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_batch_remainder_transform(self):
         warnings.filterwarnings("ignore")
         model = StandardScaler(with_mean=True, with_std=True)
@@ -521,6 +542,10 @@ def test_tvm_batch_remainder_transform(self):
 
     # Test tvm regression with batching and uneven numer of records.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_regression_remainder_batch(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -542,6 +567,10 @@ def test_tvm_regression_remainder_batch(self):
 
     # Test tvm classification with batching and uneven numer of records.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_classification_remainder_batch(self):
         warnings.filterwarnings("ignore")
         max_depth = 10
@@ -564,6 +593,10 @@ def test_tvm_classification_remainder_batch(self):
 
     # Test tvm iforest with batching and uneven numer of records.
     @unittest.skipIf(not tvm_installed(), reason="TVM test require TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_iforest_remainder_batch(self):
         warnings.filterwarnings("ignore")
         num_classes = 2
@@ -744,6 +777,10 @@ def test_pandas_batch_onnxml(self):
     # Test batch with pandas tvm.
     @unittest.skipIf(not pandas_installed(), reason="Test requires pandas installed")
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_pandas_batch_tvm(self):
         import pandas
 
@@ -801,6 +838,10 @@ def test_lightgbm_pytorch_extra_config(self):
 
     # Test max fuse depth configuration in TVM.
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_max_fuse(self):
         warnings.filterwarnings("ignore")
 
@@ -816,6 +857,10 @@ def test_tvm_max_fuse(self):
 
     # Test TVM without padding returns an errror is sizes don't match.
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_no_padding(self):
         warnings.filterwarnings("ignore")
 
@@ -832,6 +877,10 @@ def test_tvm_no_padding(self):
 
     # Test padding in TVM.
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_padding(self):
         warnings.filterwarnings("ignore")
 
@@ -848,6 +897,10 @@ def test_tvm_padding(self):
 
     # Test padding in TVM does not create problems when not necessary.
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_tvm_padding_2(self):
         warnings.filterwarnings("ignore")
 

tests/test_xgboost_converter.py

@@ -1,6 +1,7 @@
 """
 Tests XGBoost converters.
 """
+import sys
 import unittest
 import warnings
 
@@ -9,7 +10,7 @@
 from sklearn.model_selection import train_test_split
 
 import hummingbird.ml
-from hummingbird.ml._utils import xgboost_installed, tvm_installed, pandas_installed
+from hummingbird.ml._utils import xgboost_installed, tvm_installed, pandas_installed, is_on_github_actions
 from hummingbird.ml import constants
 from tree_utils import gbdt_implementation_map
 
@@ -291,6 +292,10 @@ def test_xgb_classifier_converter_torchscript(self):
     # TVM backend regression.
     @unittest.skipIf(not xgboost_installed(), reason="XGBoost test requires XGBoost installed")
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_xgb_regressor_converter_tvm(self):
         warnings.filterwarnings("ignore")
         import torch
@@ -311,6 +316,10 @@ def test_xgb_regressor_converter_tvm(self):
     # Test TVM backend classification.
     @unittest.skipIf(not xgboost_installed(), reason="XGBoost test requires XGBoost installed")
     @unittest.skipIf(not tvm_installed(), reason="TVM test requires TVM installed")
+    @unittest.skipIf(
+        ((sys.platform == "linux") and is_on_github_actions()),
+        reason="This test is flaky on Ubuntu on GitHub Actions. See https://github.com/microsoft/hummingbird/pull/709 for more info.",
+    )
     def test_xgb_classifier_converter_tvm(self):
         warnings.filterwarnings("ignore")
         import torch