pyo3-polars 0.20 -> Polars 0.46.0 (#128)

Cargo.toml

@@ -9,15 +9,18 @@ members = [
 ]
 
 [workspace.dependencies]
-polars = { version = "0.45.0", default-features = false }
-polars-core = { version = "0.45.0", default-features = false }
-polars-ffi = { version = "0.45.0", default-features = false }
-polars-plan = { version = "0.45.0", default-features = false }
-polars-lazy = { version = "0.45.0", default-features = false }
+polars = { version = "0.46.0", default-features = false }
+polars-core = { version = "0.46.0", default-features = false }
+polars-arrow = { version = "0.46.0", default-features = false }
+polars-ffi = { version = "0.46.0", default-features = false }
+polars-plan = { version = "0.46.0", default-features = false }
+polars-lazy = { version = "0.46.0", default-features = false }
+polars-python = {version = "0.46.0", default-features = false }
+polars-utils = {version = "0.46.0", default-features = false}
 
 [workspace.dependencies.arrow]
 package = "polars-arrow"
-version = "0.45.0"
+version = "0.46.0"
 path = "../polars/crates/polars-arrow"
 default-features = false
 

example/derive_expression/expression_lib/Cargo.toml

@@ -10,7 +10,9 @@ crate-type = ["cdylib"]
 
 [dependencies]
 polars = { workspace = true, features = ["fmt", "dtype-date", "timezones"], default-features = false }
-pyo3 = { version = "0.22", features = ["abi3-py38"] }
+polars-arrow = { workspace = true, default-features = false }
+pyo3 = { version = "0.23", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] }
 rayon = "1.7.0"
 serde = { version = "1", features = ["derive"] }
+num = "*"

example/derive_expression/expression_lib/src/distances.rs

@@ -1,6 +1,6 @@
-use polars::export::arrow::array::PrimitiveArray;
-use polars::export::num::Float;
+use num::Float;
 use polars::prelude::*;
+use polars_arrow::array::PrimitiveArray;
 use pyo3_polars::export::polars_core::utils::arrow::types::NativeType;
 use pyo3_polars::export::polars_core::with_match_physical_integer_type;
 use std::hash::Hash;

example/extend_polars_python_dispatch/extend_polars/Cargo.toml

@@ -12,6 +12,6 @@ crate-type = ["cdylib"]
 polars = { workspace = true, features = ["fmt"] }
 polars-core = { workspace = true }
 polars-lazy = { workspace = true }
-pyo3 = { version = "0.22", features = ["extension-module"] }
+pyo3 = { version = "0.23", features = ["extension-module"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] }
 rayon = "1.10"

example/extend_polars_python_dispatch/extend_polars/src/lib.rs

@@ -27,6 +27,7 @@ fn debug(pydf: PyDataFrame) -> PyResult<PyDataFrame> {
 #[pyfunction]
 fn lazy_parallel_jaccard(pydf: PyLazyFrame, col_a: &str, col_b: &str) -> PyResult<PyLazyFrame> {
     let df: LazyFrame = pydf.into();
+    dbg!(&df.describe_plan());
     let df = parallel_jaccard_mod::parallel_jaccard(df.collect().unwrap(), col_a, col_b)
         .map_err(PyPolarsErr::from)?;
     Ok(PyLazyFrame(df.lazy()))

example/io_plugin/io_plugin/Cargo.toml

@@ -10,6 +10,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy"], default-features = false }
-pyo3 = { version = "0.22", features = ["abi3-py38"] }
+polars-arrow = { workspace = true, default-features = false }
+pyo3 = { version = "0.23", features = ["abi3-py38"] }
 pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive", "lazy"] }
 rand = { version = "0.8.5", features = [] }

example/io_plugin/io_plugin/src/samplers.rs

@@ -1,9 +1,9 @@
-use polars::export::arrow::bitmap::MutableBitmap;
-use polars::export::arrow::types::NativeType;
 use polars::prelude::*;
+use polars_arrow::array::BooleanArray;
+use polars_arrow::bitmap::MutableBitmap;
+use polars_arrow::types::NativeType;
 use pyo3::{pyclass, pyfunction};
 use pyo3_polars::export::polars_core::datatypes::{DataType, PolarsDataType};
-use pyo3_polars::export::polars_core::export::arrow::array::BooleanArray;
 use pyo3_polars::export::polars_core::prelude::Series;
 use pyo3_polars::PyDataType;
 use rand::distributions::uniform::SampleUniform;

pyo3-polars-derive/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars-derive"
-version = "0.13.0"
+version = "0.14.0"
 edition = "2021"
 license = "MIT"
 readme = "README.md"
@@ -20,6 +20,7 @@ path = "tests/run.rs"
 polars-core = { workspace = true }
 polars-ffi = { workspace = true }
 polars-plan = { workspace = true }
+polars-arrow = { workspace = true }
 proc-macro2 = "1.0"
 quote = "1.0"
 syn = { version = "2", features = ["full", "extra-traits"] }

pyo3-polars-derive/src/lib.rs

@@ -197,7 +197,7 @@ fn quote_get_inputs() -> proc_macro2::TokenStream {
     quote!(
              let inputs = std::slice::from_raw_parts(field, len);
              let inputs = inputs.iter().map(|field| {
-                 let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap();
+                 let field = polars_arrow::ffi::import_field_from_c(field).unwrap();
                  let out = polars_core::prelude::Field::from(&field);
                  out
              }).collect::<Vec<_>>();
@@ -227,9 +227,9 @@ fn create_field_function(
     quote! (
         #[no_mangle]
         pub unsafe extern "C" fn #map_field_name(
-            field: *mut polars_core::export::arrow::ffi::ArrowSchema,
+            field: *mut polars_arrow::ffi::ArrowSchema,
             len: usize,
-            return_value: *mut polars_core::export::arrow::ffi::ArrowSchema,
+            return_value: *mut polars_arrow::ffi::ArrowSchema,
             kwargs_ptr: *const u8,
             kwargs_len: usize,
         ) {
@@ -240,7 +240,7 @@ fn create_field_function(
 
                 match result {
                     Ok(out) => {
-                        let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest()));
+                        let out = polars_arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest()));
                         *return_value = out;
                     },
                     Err(err) => {
@@ -268,16 +268,16 @@ fn create_field_function_from_with_dtype(
     quote! (
         #[no_mangle]
         pub unsafe extern "C" fn #map_field_name(
-            field: *mut polars_core::export::arrow::ffi::ArrowSchema,
+            field: *mut polars_arrow::ffi::ArrowSchema,
             len: usize,
-            return_value: *mut polars_core::export::arrow::ffi::ArrowSchema
+            return_value: *mut polars_arrow::ffi::ArrowSchema
         ) {
             #inputs
 
             let mapper = polars_plan::dsl::FieldsMapper::new(&inputs);
             let dtype = polars_core::datatypes::DataType::#dtype;
             let out = mapper.with_dtype(dtype).unwrap();
-            let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest()));
+            let out = polars_arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest()));
             *return_value = out;
         }
     )

pyo3-polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyo3-polars"
-version = "0.19.0"
+version = "0.20.0"
 edition = "2021"
 license = "MIT"
 readme = "../README.md"
@@ -10,22 +10,24 @@ description = "Expression plugins and PyO3 types for polars"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-ciborium = { version = "0.2.1", optional = true }
 libc = "0.2" # pyo3 depends on libc already, so this does not introduce an extra dependence.
 once_cell = "1"
-polars = { workspace = true, default-features = false }
+polars = { workspace = true, default-features = true}
+polars-arrow = { workspace = true, default-features = false }
 polars-core = { workspace = true, default-features = false }
 polars-ffi = { workspace = true, optional = true }
 polars-lazy = { workspace = true, optional = true }
 polars-plan = { workspace = true, optional = true }
-pyo3 = "0.22"
-pyo3-polars-derive = { version = "0.13.0", path = "../pyo3-polars-derive", optional = true }
+polars-utils = {workspace = true, features = ["serde"], optional = true }
+pyo3 = "0.23"
+pyo3-polars-derive = { version = "0.14.0", path = "../pyo3-polars-derive", optional = true }
 serde = { version = "1", optional = true }
 serde-pickle = { version = "1", optional = true }
 thiserror = "1"
 
 [features]
-lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "ciborium"]
+# Polars python is needed because all variants need to be acttivated of the DSL.
+lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "polars-utils", "polars-lazy/python"]
 derive = ["pyo3-polars-derive", "polars-plan", "polars-ffi", "serde-pickle", "serde"]
 dtype-full = ["polars/dtype-full", "dtype-decimal", "dtype-array", "dtype-struct", "dtype-categorical"]
 object = ["polars/object"]

pyo3-polars/src/ffi/to_py.rs

@@ -1,14 +1,14 @@
-use polars::export::arrow::ffi;
+use polars_arrow::ffi;
+
 use polars::prelude::{ArrayRef, ArrowField};
 use pyo3::ffi::Py_uintptr_t;
 use pyo3::prelude::*;
 
 /// Arrow array to Python.
-pub(crate) fn to_py_array(
+pub(crate) fn to_py_array<'py>(
     array: ArrayRef,
-    py: Python,
-    pyarrow: Bound<'_, PyModule>,
-) -> PyResult<PyObject> {
+    pyarrow: Bound<'py, PyModule>,
+) -> PyResult<Bound<'py, PyAny>> {
     let schema = Box::new(ffi::export_field_to_c(&ArrowField::new(
         "".into(),
         array.dtype().clone(),
@@ -19,10 +19,8 @@ pub(crate) fn to_py_array(
     let schema_ptr: *const ffi::ArrowSchema = &*schema;
     let array_ptr: *const ffi::ArrowArray = &*array;
 
-    let array = pyarrow.getattr("Array")?.call_method1(
+    pyarrow.getattr("Array")?.call_method1(
         "_import_arrow_from_c",
         (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t),
-    )?;
-
-    Ok(array.to_object(py))
+    )
 }

pyo3-polars/src/ffi/to_rust.rs

@@ -1,6 +1,6 @@
 use crate::error::PyPolarsErr;
-use polars::export::arrow::ffi;
 use polars::prelude::*;
+use polars_arrow::ffi;
 use pyo3::ffi::Py_uintptr_t;
 use pyo3::prelude::*;
 

pyo3-polars/src/lib.rs

@@ -55,9 +55,8 @@ use once_cell::sync::Lazy;
 use pyo3::prelude::*;
 pub use types::*;
 
-pub(crate) static POLARS: Lazy<PyObject> = Lazy::new(|| {
-    Python::with_gil(|py| PyModule::import_bound(py, "polars").unwrap().to_object(py))
-});
+pub(crate) static POLARS: Lazy<Py<PyModule>> =
+    Lazy::new(|| Python::with_gil(|py| PyModule::import(py, "polars").unwrap().unbind()));
 
-pub(crate) static SERIES: Lazy<PyObject> =
+pub(crate) static SERIES: Lazy<Py<PyAny>> =
     Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "Series").unwrap()));