Skip to content

Commit

Permalink
AP-16457: Add projects for fast-tables based Java/Python data transfer
Browse files Browse the repository at this point in the history
This commit adds three new projects
* org.knime.python3
    Core utilities for handling Python processes from Java. Used to
    start processes and call functions on them using Py4j. Also provides
    an extension system for wrapping Java objects providing data and
    getting data in Python objects with a more Pythonic API.
* org.knime.python3.arrow
    Utilities for getting Arrow-based KNIME data to a Python process and
    the other way around.
* org.knime.python3.arrow.tests
    Tests for org.knime.python3.arrow

These projects are work-in-progress and will evolve in the upcoming
weeks and months.

AP-16457 (Implement Java/Python data transfer based on Columnar Arrow Backend)
  • Loading branch information
HedgehogCode authored and chaubold committed Jul 1, 2021
1 parent b8e5d64 commit 7982565
Show file tree
Hide file tree
Showing 60 changed files with 7,577 additions and 0 deletions.
7 changes: 7 additions & 0 deletions org.knime.python3.arrow.tests/.classpath
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="con" path="org.eclipse.pde.core.requiredPlugins"/>
<classpathentry kind="src" path="src/test/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-11"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>
28 changes: 28 additions & 0 deletions org.knime.python3.arrow.tests/.project
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>org.knime.python3.arrow.tests</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.pde.ManifestBuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.pde.SchemaBuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.pde.PluginNature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
eclipse.preferences.version=1
encoding/<project>=UTF-8
437 changes: 437 additions & 0 deletions org.knime.python3.arrow.tests/.settings/org.eclipse.jdt.core.prefs

Large diffs are not rendered by default.

115 changes: 115 additions & 0 deletions org.knime.python3.arrow.tests/.settings/org.eclipse.jdt.ui.prefs
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
cleanup.add_default_serial_version_id=true
cleanup.add_generated_serial_version_id=false
cleanup.add_missing_annotations=true
cleanup.add_missing_deprecated_annotations=true
cleanup.add_missing_methods=false
cleanup.add_missing_nls_tags=false
cleanup.add_missing_override_annotations=true
cleanup.add_missing_override_annotations_interface_methods=true
cleanup.add_serial_version_id=false
cleanup.always_use_blocks=true
cleanup.always_use_parentheses_in_expressions=true
cleanup.always_use_this_for_non_static_field_access=false
cleanup.always_use_this_for_non_static_method_access=false
cleanup.convert_to_enhanced_for_loop=false
cleanup.correct_indentation=true
cleanup.format_source_code=false
cleanup.format_source_code_changes_only=false
cleanup.make_local_variable_final=true
cleanup.make_parameters_final=true
cleanup.make_private_fields_final=true
cleanup.make_type_abstract_if_missing_method=false
cleanup.make_variable_declarations_final=true
cleanup.never_use_blocks=false
cleanup.never_use_parentheses_in_expressions=false
cleanup.organize_imports=true
cleanup.qualify_static_field_accesses_with_declaring_class=false
cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true
cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true
cleanup.qualify_static_member_accesses_with_declaring_class=true
cleanup.qualify_static_method_accesses_with_declaring_class=false
cleanup.remove_private_constructors=true
cleanup.remove_trailing_whitespaces=true
cleanup.remove_trailing_whitespaces_all=true
cleanup.remove_trailing_whitespaces_ignore_empty=false
cleanup.remove_unnecessary_casts=true
cleanup.remove_unnecessary_nls_tags=true
cleanup.remove_unused_imports=true
cleanup.remove_unused_local_variables=false
cleanup.remove_unused_private_fields=true
cleanup.remove_unused_private_members=false
cleanup.remove_unused_private_methods=true
cleanup.remove_unused_private_types=true
cleanup.sort_members=false
cleanup.sort_members_all=false
cleanup.use_blocks=true
cleanup.use_blocks_only_for_return_and_throw=false
cleanup.use_parentheses_in_expressions=true
cleanup.use_this_for_non_static_field_access=false
cleanup.use_this_for_non_static_field_access_only_if_necessary=true
cleanup.use_this_for_non_static_method_access=false
cleanup.use_this_for_non_static_method_access_only_if_necessary=true
cleanup_profile=_KNIME
cleanup_settings_version=2
eclipse.preferences.version=1
editor_save_participant_org.eclipse.jdt.ui.postsavelistener.cleanup=true
formatter_profile=_UniKN KNIME
formatter_settings_version=12
org.eclipse.jdt.ui.ignorelowercasenames=true
org.eclipse.jdt.ui.importorder=java;javax;org;com;
org.eclipse.jdt.ui.javadoc=true
org.eclipse.jdt.ui.ondemandthreshold=99
org.eclipse.jdt.ui.staticondemandthreshold=99
org.eclipse.jdt.ui.text.custom_code_templates=<?xml version\="1.0" encoding\="UTF-8" standalone\="no"?><templates><template autoinsert\="true" context\="gettercomment_context" deleted\="false" description\="Comment for getter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.gettercomment" name\="gettercomment">/**\n * @return the ${bare_field_name}\n */</template><template autoinsert\="true" context\="settercomment_context" deleted\="false" description\="Comment for setter method" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.settercomment" name\="settercomment">/**\n * @param ${param} the ${bare_field_name} to set\n */</template><template autoinsert\="true" context\="constructorcomment_context" deleted\="false" description\="Comment for created constructors" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorcomment" name\="constructorcomment">/**\n * ${tags}\n */</template><template autoinsert\="true" context\="filecomment_context" deleted\="false" description\="Comment for created Java files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.filecomment" name\="filecomment">/**\n * \n */</template><template autoinsert\="false" context\="typecomment_context" deleted\="false" description\="Comment for created types" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.typecomment" name\="typecomment">/**\n * \n * @author ${user}\n */</template><template autoinsert\="true" context\="fieldcomment_context" deleted\="false" description\="Comment for fields" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.fieldcomment" name\="fieldcomment">/**\n * \n */</template><template autoinsert\="true" context\="methodcomment_context" deleted\="false" description\="Comment for non-overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodcomment" name\="methodcomment">/**\n * ${tags}\n */</template><template autoinsert\="false" context\="overridecomment_context" deleted\="false" description\="Comment for overriding methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.overridecomment" name\="overridecomment">/**\n * {@inheritDoc}\n */</template><template autoinsert\="true" context\="delegatecomment_context" deleted\="false" description\="Comment for delegate methods" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.delegatecomment" name\="delegatecomment">/**\n * ${tags}\n * ${see_to_target}\n */</template><template autoinsert\="false" context\="newtype_context" deleted\="false" description\="Newly created files" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.newtype" name\="newtype">/*\n * ------------------------------------------------------------------------\n *\n * Copyright by KNIME AG, Zurich, Switzerland\n * Website\: http\://www.knime.com; Email\: [email protected]\n *\n * This program is free software; you can redistribute it and/or modify\n * it under the terms of the GNU General Public License, Version 3, as\n * published by the Free Software Foundation.\n *\n * This program is distributed in the hope that it will be useful, but\n * WITHOUT ANY WARRANTY; without even the implied warranty of\n * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the\n * GNU General Public License for more details.\n *\n * You should have received a copy of the GNU General Public License\n * along with this program; if not, see &lt;http\://www.gnu.org/licenses&gt;.\n *\n * Additional permission under GNU GPL version 3 section 7\:\n *\n * KNIME interoperates with ECLIPSE solely via ECLIPSE's plug-in APIs.\n * Hence, KNIME and ECLIPSE are both independent programs and are not\n * derived from each other. Should, however, the interpretation of the\n * GNU GPL Version 3 ("License") under any applicable laws result in\n * KNIME and ECLIPSE being a combined program, KNIME AG herewith grants\n * you the additional permission to use and propagate KNIME together with\n * ECLIPSE with only the license terms in place for ECLIPSE applying to\n * ECLIPSE and the GNU GPL Version 3 applying for KNIME, provided the\n * license terms of ECLIPSE themselves allow for the respective use and\n * propagation of ECLIPSE together with KNIME.\n *\n * Additional permission relating to nodes for KNIME that extend the Node\n * Extension (and in particular that are based on subclasses of NodeModel,\n * NodeDialog, and NodeView) and that only interoperate with KNIME through\n * standard APIs ("Nodes")\:\n * Nodes are deemed to be separate and independent programs and to not be\n * covered works. Notwithstanding anything to the contrary in the\n * License, the License does not apply to Nodes, you are not required to\n * license Nodes under the License, and you are granted a license to\n * prepare and propagate Nodes, in each case even if such Nodes are\n * propagated with or for interoperation with KNIME. The owner of a Node\n * may freely choose the license terms applicable to such Node, including\n * when such Node is propagated with or for interoperation with KNIME.\n * ---------------------------------------------------------------------\n * \n * History\n * ${date} (${user})\: created\n */\n${package_declaration}\n\n${typecomment}\n${type_declaration}</template><template autoinsert\="true" context\="classbody_context" deleted\="false" description\="Code in new class type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.classbody" name\="classbody">\n</template><template autoinsert\="true" context\="interfacebody_context" deleted\="false" description\="Code in new interface type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.interfacebody" name\="interfacebody">\n</template><template autoinsert\="true" context\="enumbody_context" deleted\="false" description\="Code in new enum type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.enumbody" name\="enumbody">\n</template><template autoinsert\="true" context\="annotationbody_context" deleted\="false" description\="Code in new annotation type bodies" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.annotationbody" name\="annotationbody">\n</template><template autoinsert\="false" context\="catchblock_context" deleted\="false" description\="Code in new catch blocks" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.catchblock" name\="catchblock">// ${todo} Auto-generated catch block</template><template autoinsert\="true" context\="methodbody_context" deleted\="false" description\="Code in created method stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.methodbody" name\="methodbody">// ${todo} Auto-generated method stub\n${body_statement}</template><template autoinsert\="true" context\="constructorbody_context" deleted\="false" description\="Code in created constructor stubs" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.constructorbody" name\="constructorbody">${body_statement}\n// ${todo} Auto-generated constructor stub</template><template autoinsert\="true" context\="getterbody_context" deleted\="false" description\="Code in created getters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.getterbody" name\="getterbody">return ${field};</template><template autoinsert\="true" context\="setterbody_context" deleted\="false" description\="Code in created setters" enabled\="true" id\="org.eclipse.jdt.ui.text.codetemplates.setterbody" name\="setterbody">${field} \= ${param};</template></templates>
sp_cleanup.add_default_serial_version_id=true
sp_cleanup.add_generated_serial_version_id=false
sp_cleanup.add_missing_annotations=true
sp_cleanup.add_missing_deprecated_annotations=true
sp_cleanup.add_missing_methods=false
sp_cleanup.add_missing_nls_tags=false
sp_cleanup.add_missing_override_annotations=true
sp_cleanup.add_missing_override_annotations_interface_methods=true
sp_cleanup.add_serial_version_id=false
sp_cleanup.always_use_blocks=true
sp_cleanup.always_use_parentheses_in_expressions=false
sp_cleanup.always_use_this_for_non_static_field_access=false
sp_cleanup.always_use_this_for_non_static_method_access=false
sp_cleanup.convert_to_enhanced_for_loop=false
sp_cleanup.correct_indentation=false
sp_cleanup.format_source_code=false
sp_cleanup.format_source_code_changes_only=false
sp_cleanup.make_local_variable_final=false
sp_cleanup.make_parameters_final=true
sp_cleanup.make_private_fields_final=false
sp_cleanup.make_type_abstract_if_missing_method=false
sp_cleanup.make_variable_declarations_final=true
sp_cleanup.never_use_blocks=false
sp_cleanup.never_use_parentheses_in_expressions=true
sp_cleanup.on_save_use_additional_actions=true
sp_cleanup.organize_imports=true
sp_cleanup.qualify_static_field_accesses_with_declaring_class=false
sp_cleanup.qualify_static_member_accesses_through_instances_with_declaring_class=true
sp_cleanup.qualify_static_member_accesses_through_subtypes_with_declaring_class=true
sp_cleanup.qualify_static_member_accesses_with_declaring_class=true
sp_cleanup.qualify_static_method_accesses_with_declaring_class=false
sp_cleanup.remove_private_constructors=false
sp_cleanup.remove_trailing_whitespaces=true
sp_cleanup.remove_trailing_whitespaces_all=true
sp_cleanup.remove_trailing_whitespaces_ignore_empty=false
sp_cleanup.remove_unnecessary_casts=true
sp_cleanup.remove_unnecessary_nls_tags=false
sp_cleanup.remove_unused_imports=true
sp_cleanup.remove_unused_local_variables=false
sp_cleanup.remove_unused_private_fields=false
sp_cleanup.remove_unused_private_members=false
sp_cleanup.remove_unused_private_methods=false
sp_cleanup.remove_unused_private_types=false
sp_cleanup.sort_members=false
sp_cleanup.sort_members_all=false
sp_cleanup.use_blocks=true
sp_cleanup.use_blocks_only_for_return_and_throw=false
sp_cleanup.use_parentheses_in_expressions=false
sp_cleanup.use_this_for_non_static_field_access=false
sp_cleanup.use_this_for_non_static_field_access_only_if_necessary=true
sp_cleanup.use_this_for_non_static_method_access=false
sp_cleanup.use_this_for_non_static_method_access_only_if_necessary=true
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
ANNOTATION_ELEMENT_TYPE_ADDED_FIELD=Error
ANNOTATION_ELEMENT_TYPE_ADDED_METHOD_WITHOUT_DEFAULT_VALUE=Error
ANNOTATION_ELEMENT_TYPE_CHANGED_TYPE_CONVERSION=Error
ANNOTATION_ELEMENT_TYPE_REMOVED_FIELD=Error
ANNOTATION_ELEMENT_TYPE_REMOVED_METHOD=Error
ANNOTATION_ELEMENT_TYPE_REMOVED_TYPE_MEMBER=Error
API_COMPONENT_ELEMENT_TYPE_REMOVED_API_TYPE=Error
API_COMPONENT_ELEMENT_TYPE_REMOVED_REEXPORTED_API_TYPE=Error
API_COMPONENT_ELEMENT_TYPE_REMOVED_REEXPORTED_TYPE=Error
API_COMPONENT_ELEMENT_TYPE_REMOVED_TYPE=Error
API_USE_SCAN_FIELD_SEVERITY=Error
API_USE_SCAN_METHOD_SEVERITY=Error
API_USE_SCAN_TYPE_SEVERITY=Error
CLASS_ELEMENT_TYPE_ADDED_FIELD=Error
CLASS_ELEMENT_TYPE_ADDED_METHOD=Error
CLASS_ELEMENT_TYPE_ADDED_RESTRICTIONS=Error
CLASS_ELEMENT_TYPE_ADDED_TYPE_PARAMETER=Error
CLASS_ELEMENT_TYPE_CHANGED_CONTRACTED_SUPERINTERFACES_SET=Error
CLASS_ELEMENT_TYPE_CHANGED_DECREASE_ACCESS=Error
CLASS_ELEMENT_TYPE_CHANGED_NON_ABSTRACT_TO_ABSTRACT=Error
CLASS_ELEMENT_TYPE_CHANGED_NON_FINAL_TO_FINAL=Error
CLASS_ELEMENT_TYPE_CHANGED_TYPE_CONVERSION=Error
CLASS_ELEMENT_TYPE_REMOVED_CONSTRUCTOR=Error
CLASS_ELEMENT_TYPE_REMOVED_FIELD=Error
CLASS_ELEMENT_TYPE_REMOVED_METHOD=Error
CLASS_ELEMENT_TYPE_REMOVED_SUPERCLASS=Error
CLASS_ELEMENT_TYPE_REMOVED_TYPE_MEMBER=Error
CLASS_ELEMENT_TYPE_REMOVED_TYPE_PARAMETER=Error
CONSTRUCTOR_ELEMENT_TYPE_ADDED_TYPE_PARAMETER=Error
CONSTRUCTOR_ELEMENT_TYPE_CHANGED_DECREASE_ACCESS=Error
CONSTRUCTOR_ELEMENT_TYPE_CHANGED_VARARGS_TO_ARRAY=Error
CONSTRUCTOR_ELEMENT_TYPE_REMOVED_TYPE_PARAMETER=Error
ENUM_ELEMENT_TYPE_CHANGED_CONTRACTED_SUPERINTERFACES_SET=Error
ENUM_ELEMENT_TYPE_CHANGED_TYPE_CONVERSION=Error
ENUM_ELEMENT_TYPE_REMOVED_ENUM_CONSTANT=Error
ENUM_ELEMENT_TYPE_REMOVED_FIELD=Error
ENUM_ELEMENT_TYPE_REMOVED_METHOD=Error
ENUM_ELEMENT_TYPE_REMOVED_TYPE_MEMBER=Error
FIELD_ELEMENT_TYPE_ADDED_VALUE=Error
FIELD_ELEMENT_TYPE_CHANGED_DECREASE_ACCESS=Error
FIELD_ELEMENT_TYPE_CHANGED_FINAL_TO_NON_FINAL_STATIC_CONSTANT=Error
FIELD_ELEMENT_TYPE_CHANGED_NON_FINAL_TO_FINAL=Error
FIELD_ELEMENT_TYPE_CHANGED_NON_STATIC_TO_STATIC=Error
FIELD_ELEMENT_TYPE_CHANGED_STATIC_TO_NON_STATIC=Error
FIELD_ELEMENT_TYPE_CHANGED_TYPE=Error
FIELD_ELEMENT_TYPE_CHANGED_VALUE=Error
FIELD_ELEMENT_TYPE_REMOVED_TYPE_ARGUMENT=Error
FIELD_ELEMENT_TYPE_REMOVED_VALUE=Error
ILLEGAL_EXTEND=Warning
ILLEGAL_IMPLEMENT=Warning
ILLEGAL_INSTANTIATE=Warning
ILLEGAL_OVERRIDE=Warning
ILLEGAL_REFERENCE=Warning
INTERFACE_ELEMENT_TYPE_ADDED_DEFAULT_METHOD=Ignore
INTERFACE_ELEMENT_TYPE_ADDED_FIELD=Error
INTERFACE_ELEMENT_TYPE_ADDED_METHOD=Error
INTERFACE_ELEMENT_TYPE_ADDED_RESTRICTIONS=Error
INTERFACE_ELEMENT_TYPE_ADDED_SUPER_INTERFACE_WITH_METHODS=Error
INTERFACE_ELEMENT_TYPE_ADDED_TYPE_PARAMETER=Error
INTERFACE_ELEMENT_TYPE_CHANGED_CONTRACTED_SUPERINTERFACES_SET=Error
INTERFACE_ELEMENT_TYPE_CHANGED_TYPE_CONVERSION=Error
INTERFACE_ELEMENT_TYPE_REMOVED_FIELD=Error
INTERFACE_ELEMENT_TYPE_REMOVED_METHOD=Error
INTERFACE_ELEMENT_TYPE_REMOVED_TYPE_MEMBER=Error
INTERFACE_ELEMENT_TYPE_REMOVED_TYPE_PARAMETER=Error
INVALID_ANNOTATION=Ignore
INVALID_JAVADOC_TAG=Ignore
INVALID_REFERENCE_IN_SYSTEM_LIBRARIES=Ignore
LEAK_EXTEND=Warning
LEAK_FIELD_DECL=Warning
LEAK_IMPLEMENT=Warning
LEAK_METHOD_PARAM=Warning
LEAK_METHOD_RETURN_TYPE=Warning
METHOD_ELEMENT_TYPE_ADDED_RESTRICTIONS=Error
METHOD_ELEMENT_TYPE_ADDED_TYPE_PARAMETER=Error
METHOD_ELEMENT_TYPE_CHANGED_DECREASE_ACCESS=Error
METHOD_ELEMENT_TYPE_CHANGED_NON_ABSTRACT_TO_ABSTRACT=Error
METHOD_ELEMENT_TYPE_CHANGED_NON_FINAL_TO_FINAL=Error
METHOD_ELEMENT_TYPE_CHANGED_NON_STATIC_TO_STATIC=Error
METHOD_ELEMENT_TYPE_CHANGED_STATIC_TO_NON_STATIC=Error
METHOD_ELEMENT_TYPE_CHANGED_VARARGS_TO_ARRAY=Error
METHOD_ELEMENT_TYPE_REMOVED_ANNOTATION_DEFAULT_VALUE=Error
METHOD_ELEMENT_TYPE_REMOVED_TYPE_PARAMETER=Error
MISSING_EE_DESCRIPTIONS=Warning
TYPE_PARAMETER_ELEMENT_TYPE_ADDED_CLASS_BOUND=Error
TYPE_PARAMETER_ELEMENT_TYPE_ADDED_INTERFACE_BOUND=Error
TYPE_PARAMETER_ELEMENT_TYPE_CHANGED_CLASS_BOUND=Error
TYPE_PARAMETER_ELEMENT_TYPE_CHANGED_INTERFACE_BOUND=Error
TYPE_PARAMETER_ELEMENT_TYPE_REMOVED_CLASS_BOUND=Error
TYPE_PARAMETER_ELEMENT_TYPE_REMOVED_INTERFACE_BOUND=Error
UNUSED_PROBLEM_FILTERS=Warning
automatically_removed_unused_problem_filters=Enabled
changed_execution_env=Ignore
eclipse.preferences.version=1
incompatible_api_component_version=Error
incompatible_api_component_version_report_major_without_breaking_change=Warning
incompatible_api_component_version_report_minor_without_api_change=Warning
invalid_since_tag_version=Error
malformed_since_tag=Error
missing_since_tag=Error
report_api_breakage_when_major_version_incremented=Disabled
report_resolution_errors_api_component=Warning
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
autoEnabled=true
eclipse.preferences.version=1
idePrefixKey=
projectKey=org.knime\:knime-web
serverId=sonar.knime.com
sqPrefixKey=
Loading

0 comments on commit 7982565

Please sign in to comment.