Move Python arrow test utilites

knime · Mar 13, 2023 · ef458e8 · ef458e8
1 parent 19d764a
commit ef458e8
Show file tree

Hide file tree

Showing 12 changed files with 142 additions and 75 deletions.
diff --git a/...knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java b/...knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java
@@ -95,6 +95,7 @@
 import org.knime.core.data.columnar.table.DefaultColumnarBatchReadStore.ColumnarBatchReadStoreBuilder;
 import org.knime.core.data.columnar.table.DefaultColumnarBatchStore.ColumnarBatchStoreBuilder;
 import org.knime.core.table.schema.ColumnarSchema;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Test transfer of different Arrow types to Python. Always transfers one table with one column of a specific type.
@@ -339,8 +340,8 @@ public void testDictEncodedVarBinary() throws Exception {
     /** Test sending data to Python for the given type using the values from the valueSetter */
     private <T extends NullableWriteData> void test(final String type, final ColumnarSchema schema,
         final ValueSetter<T> valueSetter) throws Exception {
-        final var writePath = TestUtils.createTmpKNIMEArrowFileHandle();
-        final var readPath = TestUtils.createTmpKNIMEArrowFileHandle();
+        final var writePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
 
         // Open connection to Python
         try (final var pythonGateway = TestUtils.openPythonGateway()) {

diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java
@@ -77,6 +77,7 @@
 import org.knime.core.table.schema.LongDataSpec;
 import org.knime.core.table.schema.StringDataSpec;
 import org.knime.python3.arrow.TestUtils.SinkCreator;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Tests sending data to a KNIME Table and receiving data from it.
@@ -112,8 +113,8 @@ public void test() throws IOException, InterruptedException {
         final var modes = new String[] {"arrow-sentinel", "arrow", "pandas", "dict"};
 
         for (final var mode : modes) {
-            final var sourcePath = TestUtils.createTmpKNIMEArrowFileHandle();
-            final var sinkPath = TestUtils.createTmpKNIMEArrowFileHandle();
+            final var sourcePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
+            final var sinkPath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
 
             final SinkCreator sinkCreator = () -> PythonArrowDataUtils.createSink(sinkPath.asPath());
 

diff --git a/...e.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java b/...e.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java
@@ -79,6 +79,7 @@
 import org.knime.core.columnar.data.IntData.IntReadData;
 import org.knime.core.columnar.data.IntData.IntWriteData;
 import org.knime.core.table.schema.ColumnarSchema;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Test different special cases for transferring Arrow data between Java and Python.
@@ -113,7 +114,7 @@ public void after() {
      */
     @Test
     public void testExpectedSchema() throws Exception {
-        final var path = TestUtils.createTmpKNIMEArrowPath();
+        final var path = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
 
         try (final var pythonGateway = TestUtils.openPythonGateway()) {
             final var entryPoint = pythonGateway.getEntryPoint();
@@ -176,7 +177,7 @@ public void testMultipleInputsOutputs() throws Exception {
             // Create the data sinks
             final List<DefaultPythonArrowDataSink> sinks = new ArrayList<>();
             for (int i = 0; i < 5; i++) {
-                final Path p = TestUtils.createTmpKNIMEArrowPath();
+                final Path p = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
                 sinks.add(PythonArrowDataUtils.createSink(p));
             }
             entryPoint.testMultipleInputsOutputs(sources, sinks);
@@ -196,12 +197,12 @@ public void testMultipleInputsOutputs() throws Exception {
 
     private ArrowBatchStore createWriteStore() throws IOException {
         final var schema = ColumnarSchema.of(INT);
-        final var path = TestUtils.createTmpKNIMEArrowFileHandle();
+        final var path = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
         return m_storeFactory.createStore(schema, path);
     }
 
     private ArrowBatchReadStore createReadStore(final int idx) throws IOException {
-        final var path = TestUtils.createTmpKNIMEArrowPath();
+        final var path = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final var writeStore = createWriteStore()) {
             try (final var writer = writeStore.getWriter()) {
                 writeData(writer, idx);

diff --git a/...ython3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java b/...ython3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java
@@ -69,6 +69,7 @@
 import org.knime.python3.PythonDataSink;
 import org.knime.python3.PythonGateway;
 import org.knime.python3.arrow.TestUtils.ArrowTestsEntryPoint;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Tests for using the {@link DomainCalculator} with a {@link PythonDataSink}.
@@ -102,7 +103,7 @@ public void after() {
      */
     @Test
     public void testDoubleDomain() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 
@@ -126,7 +127,7 @@ public void testDoubleDomain() throws Exception {
      */
     @Test
     public void testIntDomain() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 
@@ -150,7 +151,7 @@ public void testIntDomain() throws Exception {
      */
     @Test
     public void testStringDomain() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 
@@ -173,7 +174,7 @@ public void testStringDomain() throws Exception {
      */
     @Test
     public void testCategoricalStringDomain() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 

diff --git a/...e.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java b/...e.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java
@@ -63,6 +63,7 @@
 import org.knime.python3.PythonException;
 import org.knime.python3.PythonGateway;
 import org.knime.python3.arrow.TestUtils.ArrowTestsEntryPoint;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Tests for using the {@link RowKeyChecker} with a {@link PythonDataSink}.
@@ -96,7 +97,7 @@ public void after() {
      */
     @Test
     public void testDuplicateRowKeys() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 
@@ -118,7 +119,7 @@ public void testDuplicateRowKeys() throws Exception {
      */
     @Test
     public void testNoDuplicateRowKeys() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 
@@ -140,7 +141,7 @@ public void testNoDuplicateRowKeys() throws Exception {
      */
     @Test
     public void testDuplicateRowKeysException() throws Exception {
-        final var readPath = TestUtils.createTmpKNIMEArrowPath();
+        final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
         try (final PythonGateway<ArrowTestsEntryPoint> pythonGateway = TestUtils.openPythonGateway()) {
             final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint();
 

diff --git a/...knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java b/...knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java
@@ -94,6 +94,7 @@
 import org.knime.core.columnar.data.dictencoding.DictEncodedData.DictEncodedVarBinaryReadData;
 import org.knime.core.table.schema.ColumnarSchema;
 import org.knime.core.table.schema.DataSpecs;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 
 /**
  * Test transfer of different Arrow types from Python. Always transfers one table with one column of a specific type.
@@ -356,7 +357,7 @@ private <T extends NullableReadData> void test(final String type, final DataSpec
             final var entryPoint = pythonGateway.getEntryPoint();
 
             // Define a Python data sink collecting the data
-            final var outPath = TestUtils.createTmpKNIMEArrowPath();
+            final var outPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath();
             final var dataSink = PythonArrowDataUtils.createSink(outPath);
 
             // Call Python

diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java
@@ -48,16 +48,11 @@
  */
 package org.knime.python3.arrow;
 
-import java.io.File;
 import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.List;
-import java.util.UUID;
 
-import org.knime.core.columnar.store.FileHandle;
 import org.knime.python3.DefaultPythonGateway;
 import org.knime.python3.Python3SourceDirectory;
 import org.knime.python3.PythonCommand;
@@ -82,50 +77,6 @@ private TestUtils() {
         // Static utility class
     }
 
-    /**
-     * Create a temporary file which is deleted on exit.
-     *
-     * @return the file
-     * @throws IOException if the file could not be created
-     */
-    public static Path createTmpKNIMEArrowPath() throws IOException {
-        final Path path = Files.createTempFile("KNIME-" + UUID.randomUUID().toString(), ".knarrow");
-        path.toFile().deleteOnExit();
-        return path;
-    }
-
-    /**
-     * Create FileHandle that is backed by a temporary file which is deleted on exit.
-     *
-     * @return the FileHandle
-     * @throws IOException if the temporary file could not be created
-     */
-    public static FileHandle createTmpKNIMEArrowFileHandle() throws IOException {
-        final var path = createTmpKNIMEArrowPath();
-        return new FileHandle() {
-
-            @Override
-            public void delete() {
-                try {
-                    Files.deleteIfExists(path);
-                } catch (IOException e) {
-                    throw new IllegalStateException(e);
-                }
-            }
-
-            @Override
-            public Path asPath() {
-                return path;
-            }
-
-            @Override
-            public File asFile() {
-                return path.toFile();
-            }
-
-        };
-    }
-
     /**
      * @return a new {@link PythonGateway} for running tests.
      * @throws IOException
@@ -151,7 +102,7 @@ public static PythonGateway<ArrowTestsEntryPoint> openPythonGateway() throws IOE
     public interface SinkCreator {
         /**
          * @return a new data sink
-         * **/
+         **/
         PythonDataSink createSink();
     }
 
@@ -215,12 +166,14 @@ void testMultipleInputsOutputs(List<? extends PythonDataSource> dataSources,
 
         /**
          * Test the Python KNIME Table API
+         *
          * @param source providing data to Python
          * @param sinkCreator A sink supplier that is called whenever a new data sink is created on the python side
          * @param numRows number of rows in the input table
          * @param numColumns number of columns in the input table
          * @param mode Of copying data from source to sink, "arrow" or "pandas"
          */
-        PythonDataSink testKnimeTable(PythonDataSource source, SinkCreator sinkCreator, long numRows, long numColumns, String mode);
+        PythonDataSink testKnimeTable(PythonDataSource source, SinkCreator sinkCreator, long numRows, long numColumns,
+            String mode);
     }
 }
diff --git a/...types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java b/...types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java
@@ -53,8 +53,8 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.knime.core.table.schema.DataSpecs.STRING;
-import static org.knime.python3.arrow.TestUtils.createTmpKNIMEArrowFileHandle;
-import static org.knime.python3.arrow.TestUtils.createTmpKNIMEArrowPath;
+import static org.knime.python3.testing.Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle;
+import static org.knime.python3.testing.Python3ArrowTestUtils.createTmpKNIMEArrowPath;
 
 import java.io.IOException;
 import java.nio.file.Paths;
@@ -185,7 +185,7 @@
 import org.knime.python3.arrow.PythonArrowDataUtils;
 import org.knime.python3.arrow.PythonArrowDataUtils.TableDomainAndMetadata;
 import org.knime.python3.arrow.PythonArrowExtension;
-import org.knime.python3.arrow.TestUtils;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 import org.knime.python3.testing.Python3TestUtils;
 import org.knime.python3.types.PythonModule;
 import org.knime.python3.types.PythonValueFactoryModule;
@@ -736,7 +736,7 @@ public void close() throws Exception {
 
 		void runJavaToPythonTest(final ColumnarSchema schema, final DataPreparer preparer, final DataTester<E> tester)
 				throws IOException {
-			final var writePath = TestUtils.createTmpKNIMEArrowFileHandle();
+			final var writePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
 			try (final var store = m_storeFactory.createStore(schema, writePath)) {
 				try (final BatchWriter writer = store.getWriter()) {
 					preparer.writeBatch(writer);

diff --git a/org.knime.python3.arrow/META-INF/MANIFEST.MF b/org.knime.python3.arrow/META-INF/MANIFEST.MF
@@ -25,4 +25,3 @@ Export-Package: org.knime.python3.arrow
 Automatic-Module-Name: org.knime.python3.arrow
 Eclipse-BundleShape: dir
 Eclipse-RegisterBuddy: org.knime.ext.py4j
-Eclipse-ExtensibleAPI: true
diff --git a/...ipting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java b/...ipting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java
@@ -93,7 +93,7 @@
 import org.knime.python3.arrow.PythonArrowDataSource;
 import org.knime.python3.arrow.PythonArrowDataUtils;
 import org.knime.python3.arrow.PythonArrowExtension;
-import org.knime.python3.arrow.TestUtils;
+import org.knime.python3.testing.Python3ArrowTestUtils;
 import org.knime.python3.testing.Python3TestUtils;
 import org.knime.python3.views.Python3ViewsSourceDirectory;
 
@@ -227,7 +227,7 @@ private static PythonGateway<Python3KernelBackendProxyTestRunner> openPythonGate
     }
 
     private ArrowBatchReadStore createTestStore() throws IOException {
-        final var storePath = TestUtils.createTmpKNIMEArrowFileHandle();
+        final var storePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle();
         try (final ArrowBatchStore store = m_storeFactory.createStore(COLUMN_SCHEMA, storePath)) {
             try (final BatchWriter writer = store.getWriter()) {
                 for (int b = 0; b < NUM_BATCHES; b++) {

diff --git a/org.knime.python3.testing/META-INF/MANIFEST.MF b/org.knime.python3.testing/META-INF/MANIFEST.MF
@@ -7,6 +7,7 @@ Bundle-ClassPath: .
 Bundle-Vendor: KNIME AG, Zurich, Switzerland
 Bundle-RequiredExecutionEnvironment: JavaSE-11
 Require-Bundle: org.apache.commons.lang3;bundle-version="[3.9.0,4.0.0)",
- org.knime.python3;bundle-version="[5.1.0,6.0.0)"
+ org.knime.python3;bundle-version="[5.1.0,6.0.0)",
+ org.knime.core.columnar;bundle-version="[5.1.0,6.0.0)"
 Automatic-Module-Name: org.knime.python3.testing
 Export-Package: org.knime.python3.testing