diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java index d630067f7..9d98fc291 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/JavaToPythonTypeTest.java @@ -95,6 +95,7 @@ import org.knime.core.data.columnar.table.DefaultColumnarBatchReadStore.ColumnarBatchReadStoreBuilder; import org.knime.core.data.columnar.table.DefaultColumnarBatchStore.ColumnarBatchStoreBuilder; import org.knime.core.table.schema.ColumnarSchema; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Test transfer of different Arrow types to Python. Always transfers one table with one column of a specific type. @@ -339,8 +340,8 @@ public void testDictEncodedVarBinary() throws Exception { /** Test sending data to Python for the given type using the values from the valueSetter */ private void test(final String type, final ColumnarSchema schema, final ValueSetter valueSetter) throws Exception { - final var writePath = TestUtils.createTmpKNIMEArrowFileHandle(); - final var readPath = TestUtils.createTmpKNIMEArrowFileHandle(); + final var writePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); // Open connection to Python try (final var pythonGateway = TestUtils.openPythonGateway()) { diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java index 72e395267..b90bc3df6 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/KnimeTableTest.java @@ -77,6 +77,7 @@ import org.knime.core.table.schema.LongDataSpec; import org.knime.core.table.schema.StringDataSpec; import org.knime.python3.arrow.TestUtils.SinkCreator; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Tests sending data to a KNIME Table and receiving data from it. @@ -112,8 +113,8 @@ public void test() throws IOException, InterruptedException { final var modes = new String[] {"arrow-sentinel", "arrow", "pandas", "dict"}; for (final var mode : modes) { - final var sourcePath = TestUtils.createTmpKNIMEArrowFileHandle(); - final var sinkPath = TestUtils.createTmpKNIMEArrowFileHandle(); + final var sourcePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); + final var sinkPath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); final SinkCreator sinkCreator = () -> PythonArrowDataUtils.createSink(sinkPath.asPath()); diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java index 8ab960d3e..9a3e9a790 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonArrowDataUtilsTest.java @@ -79,6 +79,7 @@ import org.knime.core.columnar.data.IntData.IntReadData; import org.knime.core.columnar.data.IntData.IntWriteData; import org.knime.core.table.schema.ColumnarSchema; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Test different special cases for transferring Arrow data between Java and Python. @@ -113,7 +114,7 @@ public void after() { */ @Test public void testExpectedSchema() throws Exception { - final var path = TestUtils.createTmpKNIMEArrowPath(); + final var path = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final var pythonGateway = TestUtils.openPythonGateway()) { final var entryPoint = pythonGateway.getEntryPoint(); @@ -176,7 +177,7 @@ public void testMultipleInputsOutputs() throws Exception { // Create the data sinks final List sinks = new ArrayList<>(); for (int i = 0; i < 5; i++) { - final Path p = TestUtils.createTmpKNIMEArrowPath(); + final Path p = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); sinks.add(PythonArrowDataUtils.createSink(p)); } entryPoint.testMultipleInputsOutputs(sources, sinks); @@ -196,12 +197,12 @@ public void testMultipleInputsOutputs() throws Exception { private ArrowBatchStore createWriteStore() throws IOException { final var schema = ColumnarSchema.of(INT); - final var path = TestUtils.createTmpKNIMEArrowFileHandle(); + final var path = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); return m_storeFactory.createStore(schema, path); } private ArrowBatchReadStore createReadStore(final int idx) throws IOException { - final var path = TestUtils.createTmpKNIMEArrowPath(); + final var path = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final var writeStore = createWriteStore()) { try (final var writer = writeStore.getWriter()) { writeData(writer, idx); diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java index 245aed9b6..ecf77d43e 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonDomainCalculationTest.java @@ -69,6 +69,7 @@ import org.knime.python3.PythonDataSink; import org.knime.python3.PythonGateway; import org.knime.python3.arrow.TestUtils.ArrowTestsEntryPoint; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Tests for using the {@link DomainCalculator} with a {@link PythonDataSink}. @@ -102,7 +103,7 @@ public void after() { */ @Test public void testDoubleDomain() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); @@ -126,7 +127,7 @@ public void testDoubleDomain() throws Exception { */ @Test public void testIntDomain() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); @@ -150,7 +151,7 @@ public void testIntDomain() throws Exception { */ @Test public void testStringDomain() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); @@ -173,7 +174,7 @@ public void testStringDomain() throws Exception { */ @Test public void testCategoricalStringDomain() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java index ba103d20f..b1bd3d873 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonRowKeyCheckingTest.java @@ -63,6 +63,7 @@ import org.knime.python3.PythonException; import org.knime.python3.PythonGateway; import org.knime.python3.arrow.TestUtils.ArrowTestsEntryPoint; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Tests for using the {@link RowKeyChecker} with a {@link PythonDataSink}. @@ -96,7 +97,7 @@ public void after() { */ @Test public void testDuplicateRowKeys() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); @@ -118,7 +119,7 @@ public void testDuplicateRowKeys() throws Exception { */ @Test public void testNoDuplicateRowKeys() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); @@ -140,7 +141,7 @@ public void testNoDuplicateRowKeys() throws Exception { */ @Test public void testDuplicateRowKeysException() throws Exception { - final var readPath = TestUtils.createTmpKNIMEArrowPath(); + final var readPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); try (final PythonGateway pythonGateway = TestUtils.openPythonGateway()) { final ArrowTestsEntryPoint entryPoint = pythonGateway.getEntryPoint(); diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java index 5c62a0adb..86fc9e341 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/PythonToJavaTypeTest.java @@ -94,6 +94,7 @@ import org.knime.core.columnar.data.dictencoding.DictEncodedData.DictEncodedVarBinaryReadData; import org.knime.core.table.schema.ColumnarSchema; import org.knime.core.table.schema.DataSpecs; +import org.knime.python3.testing.Python3ArrowTestUtils; /** * Test transfer of different Arrow types from Python. Always transfers one table with one column of a specific type. @@ -356,7 +357,7 @@ private void test(final String type, final DataSpec final var entryPoint = pythonGateway.getEntryPoint(); // Define a Python data sink collecting the data - final var outPath = TestUtils.createTmpKNIMEArrowPath(); + final var outPath = Python3ArrowTestUtils.createTmpKNIMEArrowPath(); final var dataSink = PythonArrowDataUtils.createSink(outPath); // Call Python diff --git a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java index 154f92f53..81c5497d7 100644 --- a/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java +++ b/org.knime.python3.arrow.tests/src/test/java/org/knime/python3/arrow/TestUtils.java @@ -48,16 +48,11 @@ */ package org.knime.python3.arrow; -import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; import java.util.List; -import java.util.UUID; -import org.knime.core.columnar.store.FileHandle; import org.knime.python3.DefaultPythonGateway; import org.knime.python3.Python3SourceDirectory; import org.knime.python3.PythonCommand; @@ -82,50 +77,6 @@ private TestUtils() { // Static utility class } - /** - * Create a temporary file which is deleted on exit. - * - * @return the file - * @throws IOException if the file could not be created - */ - public static Path createTmpKNIMEArrowPath() throws IOException { - final Path path = Files.createTempFile("KNIME-" + UUID.randomUUID().toString(), ".knarrow"); - path.toFile().deleteOnExit(); - return path; - } - - /** - * Create FileHandle that is backed by a temporary file which is deleted on exit. - * - * @return the FileHandle - * @throws IOException if the temporary file could not be created - */ - public static FileHandle createTmpKNIMEArrowFileHandle() throws IOException { - final var path = createTmpKNIMEArrowPath(); - return new FileHandle() { - - @Override - public void delete() { - try { - Files.deleteIfExists(path); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - @Override - public Path asPath() { - return path; - } - - @Override - public File asFile() { - return path.toFile(); - } - - }; - } - /** * @return a new {@link PythonGateway} for running tests. * @throws IOException @@ -151,7 +102,7 @@ public static PythonGateway openPythonGateway() throws IOE public interface SinkCreator { /** * @return a new data sink - * **/ + **/ PythonDataSink createSink(); } @@ -215,12 +166,14 @@ void testMultipleInputsOutputs(List dataSources, /** * Test the Python KNIME Table API + * * @param source providing data to Python * @param sinkCreator A sink supplier that is called whenever a new data sink is created on the python side * @param numRows number of rows in the input table * @param numColumns number of columns in the input table * @param mode Of copying data from source to sink, "arrow" or "pandas" */ - PythonDataSink testKnimeTable(PythonDataSource source, SinkCreator sinkCreator, long numRows, long numColumns, String mode); + PythonDataSink testKnimeTable(PythonDataSource source, SinkCreator sinkCreator, long numRows, long numColumns, + String mode); } } diff --git a/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java b/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java index cd47f95f2..91dd4b024 100644 --- a/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java +++ b/org.knime.python3.arrow.types.tests/src/test/java/org/knime/python3/arrow/types/KnimeArrowExtensionTypesTest.java @@ -53,8 +53,8 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.knime.core.table.schema.DataSpecs.STRING; -import static org.knime.python3.arrow.TestUtils.createTmpKNIMEArrowFileHandle; -import static org.knime.python3.arrow.TestUtils.createTmpKNIMEArrowPath; +import static org.knime.python3.testing.Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle; +import static org.knime.python3.testing.Python3ArrowTestUtils.createTmpKNIMEArrowPath; import java.io.IOException; import java.nio.file.Paths; @@ -185,7 +185,7 @@ import org.knime.python3.arrow.PythonArrowDataUtils; import org.knime.python3.arrow.PythonArrowDataUtils.TableDomainAndMetadata; import org.knime.python3.arrow.PythonArrowExtension; -import org.knime.python3.arrow.TestUtils; +import org.knime.python3.testing.Python3ArrowTestUtils; import org.knime.python3.testing.Python3TestUtils; import org.knime.python3.types.PythonModule; import org.knime.python3.types.PythonValueFactoryModule; @@ -736,7 +736,7 @@ public void close() throws Exception { void runJavaToPythonTest(final ColumnarSchema schema, final DataPreparer preparer, final DataTester tester) throws IOException { - final var writePath = TestUtils.createTmpKNIMEArrowFileHandle(); + final var writePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); try (final var store = m_storeFactory.createStore(schema, writePath)) { try (final BatchWriter writer = store.getWriter()) { preparer.writeBatch(writer); diff --git a/org.knime.python3.arrow/META-INF/MANIFEST.MF b/org.knime.python3.arrow/META-INF/MANIFEST.MF index e92500c2c..34cafde0b 100644 --- a/org.knime.python3.arrow/META-INF/MANIFEST.MF +++ b/org.knime.python3.arrow/META-INF/MANIFEST.MF @@ -25,4 +25,3 @@ Export-Package: org.knime.python3.arrow Automatic-Module-Name: org.knime.python3.arrow Eclipse-BundleShape: dir Eclipse-RegisterBuddy: org.knime.ext.py4j -Eclipse-ExtensibleAPI: true diff --git a/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java b/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java index 35c72e2d8..2b6f2699c 100644 --- a/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java +++ b/org.knime.python3.scripting.tests/src/test/java/org/knime/python3/scripting/Python3KernelBackendProxyTest.java @@ -93,7 +93,7 @@ import org.knime.python3.arrow.PythonArrowDataSource; import org.knime.python3.arrow.PythonArrowDataUtils; import org.knime.python3.arrow.PythonArrowExtension; -import org.knime.python3.arrow.TestUtils; +import org.knime.python3.testing.Python3ArrowTestUtils; import org.knime.python3.testing.Python3TestUtils; import org.knime.python3.views.Python3ViewsSourceDirectory; @@ -227,7 +227,7 @@ private static PythonGateway openPythonGate } private ArrowBatchReadStore createTestStore() throws IOException { - final var storePath = TestUtils.createTmpKNIMEArrowFileHandle(); + final var storePath = Python3ArrowTestUtils.createTmpKNIMEArrowFileHandle(); try (final ArrowBatchStore store = m_storeFactory.createStore(COLUMN_SCHEMA, storePath)) { try (final BatchWriter writer = store.getWriter()) { for (int b = 0; b < NUM_BATCHES; b++) { diff --git a/org.knime.python3.testing/META-INF/MANIFEST.MF b/org.knime.python3.testing/META-INF/MANIFEST.MF index 75f51b51e..5fd7c2778 100644 --- a/org.knime.python3.testing/META-INF/MANIFEST.MF +++ b/org.knime.python3.testing/META-INF/MANIFEST.MF @@ -7,6 +7,7 @@ Bundle-ClassPath: . Bundle-Vendor: KNIME AG, Zurich, Switzerland Bundle-RequiredExecutionEnvironment: JavaSE-11 Require-Bundle: org.apache.commons.lang3;bundle-version="[3.9.0,4.0.0)", - org.knime.python3;bundle-version="[5.1.0,6.0.0)" + org.knime.python3;bundle-version="[5.1.0,6.0.0)", + org.knime.core.columnar;bundle-version="[5.1.0,6.0.0)" Automatic-Module-Name: org.knime.python3.testing Export-Package: org.knime.python3.testing diff --git a/org.knime.python3.testing/src/test/java/org/knime/python3/testing/Python3ArrowTestUtils.java b/org.knime.python3.testing/src/test/java/org/knime/python3/testing/Python3ArrowTestUtils.java new file mode 100644 index 000000000..43f28bc12 --- /dev/null +++ b/org.knime.python3.testing/src/test/java/org/knime/python3/testing/Python3ArrowTestUtils.java @@ -0,0 +1,108 @@ +/* + * ------------------------------------------------------------------------ + * + * Copyright by KNIME AG, Zurich, Switzerland + * Website: http://www.knime.com; Email: contact@knime.com + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License, Version 3, as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, see . + * + * Additional permission under GNU GPL version 3 section 7: + * + * KNIME interoperates with ECLIPSE solely via ECLIPSE's plug-in APIs. + * Hence, KNIME and ECLIPSE are both independent programs and are not + * derived from each other. Should, however, the interpretation of the + * GNU GPL Version 3 ("License") under any applicable laws result in + * KNIME and ECLIPSE being a combined program, KNIME AG herewith grants + * you the additional permission to use and propagate KNIME together with + * ECLIPSE with only the license terms in place for ECLIPSE applying to + * ECLIPSE and the GNU GPL Version 3 applying for KNIME, provided the + * license terms of ECLIPSE themselves allow for the respective use and + * propagation of ECLIPSE together with KNIME. + * + * Additional permission relating to nodes for KNIME that extend the Node + * Extension (and in particular that are based on subclasses of NodeModel, + * NodeDialog, and NodeView) and that only interoperate with KNIME through + * standard APIs ("Nodes"): + * Nodes are deemed to be separate and independent programs and to not be + * covered works. Notwithstanding anything to the contrary in the + * License, the License does not apply to Nodes, you are not required to + * license Nodes under the License, and you are granted a license to + * prepare and propagate Nodes, in each case even if such Nodes are + * propagated with or for interoperation with KNIME. The owner of a Node + * may freely choose the license terms applicable to such Node, including + * when such Node is propagated with or for interoperation with KNIME. + * --------------------------------------------------------------------- + * + * History + * Mar 13, 2023 (benjamin): created + */ +package org.knime.python3.testing; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.knime.core.columnar.store.FileHandle; + +/** + * Utilities for handling Arrow files in tests. + * + * @author Benjamin Wilhelm, KNIME GmbH, Konstanz, Germany + */ +public class Python3ArrowTestUtils { + + /** + * Create a temporary file which is deleted on exit. + * + * @return the file + * @throws IOException if the file could not be created + */ + public static Path createTmpKNIMEArrowPath() throws IOException { + final Path path = Files.createTempFile("KNIME-" + UUID.randomUUID().toString(), ".knarrow"); + path.toFile().deleteOnExit(); + return path; + } + + /** + * Create FileHandle that is backed by a temporary file which is deleted on exit. + * + * @return the FileHandle + * @throws IOException if the temporary file could not be created + */ + public static FileHandle createTmpKNIMEArrowFileHandle() throws IOException { + final var path = createTmpKNIMEArrowPath(); + return new FileHandle() { + + @Override + public void delete() { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + + @Override + public Path asPath() { + return path; + } + + @Override + public File asFile() { + return path.toFile(); + } + + }; + } +}