diff --git a/Cargo.toml b/Cargo.toml index 3cfe545..eda2ec4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,15 +9,18 @@ members = [ ] [workspace.dependencies] -polars = { version = "0.45.0", default-features = false } -polars-core = { version = "0.45.0", default-features = false } -polars-ffi = { version = "0.45.0", default-features = false } -polars-plan = { version = "0.45.0", default-features = false } -polars-lazy = { version = "0.45.0", default-features = false } +polars = { version = "0.46.0", default-features = false } +polars-core = { version = "0.46.0", default-features = false } +polars-arrow = { version = "0.46.0", default-features = false } +polars-ffi = { version = "0.46.0", default-features = false } +polars-plan = { version = "0.46.0", default-features = false } +polars-lazy = { version = "0.46.0", default-features = false } +polars-python = {version = "0.46.0", default-features = false } +polars-utils = {version = "0.46.0", default-features = false} [workspace.dependencies.arrow] package = "polars-arrow" -version = "0.45.0" +version = "0.46.0" path = "../polars/crates/polars-arrow" default-features = false diff --git a/example/derive_expression/expression_lib/Cargo.toml b/example/derive_expression/expression_lib/Cargo.toml index 017418b..f706e5b 100644 --- a/example/derive_expression/expression_lib/Cargo.toml +++ b/example/derive_expression/expression_lib/Cargo.toml @@ -10,7 +10,9 @@ crate-type = ["cdylib"] [dependencies] polars = { workspace = true, features = ["fmt", "dtype-date", "timezones"], default-features = false } -pyo3 = { version = "0.22", features = ["abi3-py38"] } +polars-arrow = { workspace = true, default-features = false } +pyo3 = { version = "0.23", features = ["abi3-py38"] } pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive"] } rayon = "1.7.0" serde = { version = "1", features = ["derive"] } +num = "*" diff --git a/example/derive_expression/expression_lib/src/distances.rs b/example/derive_expression/expression_lib/src/distances.rs index 2ed215f..0be0c52 100644 --- a/example/derive_expression/expression_lib/src/distances.rs +++ b/example/derive_expression/expression_lib/src/distances.rs @@ -1,6 +1,6 @@ -use polars::export::arrow::array::PrimitiveArray; -use polars::export::num::Float; +use num::Float; use polars::prelude::*; +use polars_arrow::array::PrimitiveArray; use pyo3_polars::export::polars_core::utils::arrow::types::NativeType; use pyo3_polars::export::polars_core::with_match_physical_integer_type; use std::hash::Hash; diff --git a/example/extend_polars_python_dispatch/extend_polars/Cargo.toml b/example/extend_polars_python_dispatch/extend_polars/Cargo.toml index 4a6ee08..2c4b236 100644 --- a/example/extend_polars_python_dispatch/extend_polars/Cargo.toml +++ b/example/extend_polars_python_dispatch/extend_polars/Cargo.toml @@ -12,6 +12,6 @@ crate-type = ["cdylib"] polars = { workspace = true, features = ["fmt"] } polars-core = { workspace = true } polars-lazy = { workspace = true } -pyo3 = { version = "0.22", features = ["extension-module"] } +pyo3 = { version = "0.23", features = ["extension-module"] } pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["lazy"] } rayon = "1.10" diff --git a/example/extend_polars_python_dispatch/extend_polars/src/lib.rs b/example/extend_polars_python_dispatch/extend_polars/src/lib.rs index 306cfcb..7bcc5da 100644 --- a/example/extend_polars_python_dispatch/extend_polars/src/lib.rs +++ b/example/extend_polars_python_dispatch/extend_polars/src/lib.rs @@ -27,6 +27,7 @@ fn debug(pydf: PyDataFrame) -> PyResult { #[pyfunction] fn lazy_parallel_jaccard(pydf: PyLazyFrame, col_a: &str, col_b: &str) -> PyResult { let df: LazyFrame = pydf.into(); + dbg!(&df.describe_plan()); let df = parallel_jaccard_mod::parallel_jaccard(df.collect().unwrap(), col_a, col_b) .map_err(PyPolarsErr::from)?; Ok(PyLazyFrame(df.lazy())) diff --git a/example/io_plugin/io_plugin/Cargo.toml b/example/io_plugin/io_plugin/Cargo.toml index 6312d19..3eecacc 100644 --- a/example/io_plugin/io_plugin/Cargo.toml +++ b/example/io_plugin/io_plugin/Cargo.toml @@ -10,6 +10,7 @@ crate-type = ["cdylib"] [dependencies] polars = { workspace = true, features = ["fmt", "dtype-date", "timezones", "lazy"], default-features = false } -pyo3 = { version = "0.22", features = ["abi3-py38"] } +polars-arrow = { workspace = true, default-features = false } +pyo3 = { version = "0.23", features = ["abi3-py38"] } pyo3-polars = { version = "*", path = "../../../pyo3-polars", features = ["derive", "lazy"] } rand = { version = "0.8.5", features = [] } diff --git a/example/io_plugin/io_plugin/src/samplers.rs b/example/io_plugin/io_plugin/src/samplers.rs index 1760b87..f9066e9 100644 --- a/example/io_plugin/io_plugin/src/samplers.rs +++ b/example/io_plugin/io_plugin/src/samplers.rs @@ -1,9 +1,9 @@ -use polars::export::arrow::bitmap::MutableBitmap; -use polars::export::arrow::types::NativeType; use polars::prelude::*; +use polars_arrow::array::BooleanArray; +use polars_arrow::bitmap::MutableBitmap; +use polars_arrow::types::NativeType; use pyo3::{pyclass, pyfunction}; use pyo3_polars::export::polars_core::datatypes::{DataType, PolarsDataType}; -use pyo3_polars::export::polars_core::export::arrow::array::BooleanArray; use pyo3_polars::export::polars_core::prelude::Series; use pyo3_polars::PyDataType; use rand::distributions::uniform::SampleUniform; diff --git a/pyo3-polars-derive/Cargo.toml b/pyo3-polars-derive/Cargo.toml index 7d34c9e..22195f9 100644 --- a/pyo3-polars-derive/Cargo.toml +++ b/pyo3-polars-derive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-polars-derive" -version = "0.13.0" +version = "0.14.0" edition = "2021" license = "MIT" readme = "README.md" @@ -20,6 +20,7 @@ path = "tests/run.rs" polars-core = { workspace = true } polars-ffi = { workspace = true } polars-plan = { workspace = true } +polars-arrow = { workspace = true } proc-macro2 = "1.0" quote = "1.0" syn = { version = "2", features = ["full", "extra-traits"] } diff --git a/pyo3-polars-derive/src/lib.rs b/pyo3-polars-derive/src/lib.rs index fbf4831..5e116b8 100644 --- a/pyo3-polars-derive/src/lib.rs +++ b/pyo3-polars-derive/src/lib.rs @@ -197,7 +197,7 @@ fn quote_get_inputs() -> proc_macro2::TokenStream { quote!( let inputs = std::slice::from_raw_parts(field, len); let inputs = inputs.iter().map(|field| { - let field = polars_core::export::arrow::ffi::import_field_from_c(field).unwrap(); + let field = polars_arrow::ffi::import_field_from_c(field).unwrap(); let out = polars_core::prelude::Field::from(&field); out }).collect::>(); @@ -227,9 +227,9 @@ fn create_field_function( quote! ( #[no_mangle] pub unsafe extern "C" fn #map_field_name( - field: *mut polars_core::export::arrow::ffi::ArrowSchema, + field: *mut polars_arrow::ffi::ArrowSchema, len: usize, - return_value: *mut polars_core::export::arrow::ffi::ArrowSchema, + return_value: *mut polars_arrow::ffi::ArrowSchema, kwargs_ptr: *const u8, kwargs_len: usize, ) { @@ -240,7 +240,7 @@ fn create_field_function( match result { Ok(out) => { - let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest())); + let out = polars_arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest())); *return_value = out; }, Err(err) => { @@ -268,16 +268,16 @@ fn create_field_function_from_with_dtype( quote! ( #[no_mangle] pub unsafe extern "C" fn #map_field_name( - field: *mut polars_core::export::arrow::ffi::ArrowSchema, + field: *mut polars_arrow::ffi::ArrowSchema, len: usize, - return_value: *mut polars_core::export::arrow::ffi::ArrowSchema + return_value: *mut polars_arrow::ffi::ArrowSchema ) { #inputs let mapper = polars_plan::dsl::FieldsMapper::new(&inputs); let dtype = polars_core::datatypes::DataType::#dtype; let out = mapper.with_dtype(dtype).unwrap(); - let out = polars_core::export::arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest())); + let out = polars_arrow::ffi::export_field_to_c(&out.to_arrow(CompatLevel::newest())); *return_value = out; } ) diff --git a/pyo3-polars/Cargo.toml b/pyo3-polars/Cargo.toml index a19f796..8c85f2a 100644 --- a/pyo3-polars/Cargo.toml +++ b/pyo3-polars/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-polars" -version = "0.19.0" +version = "0.20.0" edition = "2021" license = "MIT" readme = "../README.md" @@ -10,22 +10,24 @@ description = "Expression plugins and PyO3 types for polars" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -ciborium = { version = "0.2.1", optional = true } libc = "0.2" # pyo3 depends on libc already, so this does not introduce an extra dependence. once_cell = "1" -polars = { workspace = true, default-features = false } +polars = { workspace = true, default-features = true} +polars-arrow = { workspace = true, default-features = false } polars-core = { workspace = true, default-features = false } polars-ffi = { workspace = true, optional = true } polars-lazy = { workspace = true, optional = true } polars-plan = { workspace = true, optional = true } -pyo3 = "0.22" -pyo3-polars-derive = { version = "0.13.0", path = "../pyo3-polars-derive", optional = true } +polars-utils = {workspace = true, features = ["serde"], optional = true } +pyo3 = "0.23" +pyo3-polars-derive = { version = "0.14.0", path = "../pyo3-polars-derive", optional = true } serde = { version = "1", optional = true } serde-pickle = { version = "1", optional = true } thiserror = "1" [features] -lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "ciborium"] +# Polars python is needed because all variants need to be acttivated of the DSL. +lazy = ["polars/serde-lazy", "polars-plan", "polars-lazy/serde", "polars-utils", "polars-lazy/python"] derive = ["pyo3-polars-derive", "polars-plan", "polars-ffi", "serde-pickle", "serde"] dtype-full = ["polars/dtype-full", "dtype-decimal", "dtype-array", "dtype-struct", "dtype-categorical"] object = ["polars/object"] diff --git a/pyo3-polars/src/ffi/to_py.rs b/pyo3-polars/src/ffi/to_py.rs index 4070d81..a1bb283 100644 --- a/pyo3-polars/src/ffi/to_py.rs +++ b/pyo3-polars/src/ffi/to_py.rs @@ -1,14 +1,14 @@ -use polars::export::arrow::ffi; +use polars_arrow::ffi; + use polars::prelude::{ArrayRef, ArrowField}; use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; /// Arrow array to Python. -pub(crate) fn to_py_array( +pub(crate) fn to_py_array<'py>( array: ArrayRef, - py: Python, - pyarrow: Bound<'_, PyModule>, -) -> PyResult { + pyarrow: Bound<'py, PyModule>, +) -> PyResult> { let schema = Box::new(ffi::export_field_to_c(&ArrowField::new( "".into(), array.dtype().clone(), @@ -19,10 +19,8 @@ pub(crate) fn to_py_array( let schema_ptr: *const ffi::ArrowSchema = &*schema; let array_ptr: *const ffi::ArrowArray = &*array; - let array = pyarrow.getattr("Array")?.call_method1( + pyarrow.getattr("Array")?.call_method1( "_import_arrow_from_c", (array_ptr as Py_uintptr_t, schema_ptr as Py_uintptr_t), - )?; - - Ok(array.to_object(py)) + ) } diff --git a/pyo3-polars/src/ffi/to_rust.rs b/pyo3-polars/src/ffi/to_rust.rs index 6d3bd6a..c7664c1 100644 --- a/pyo3-polars/src/ffi/to_rust.rs +++ b/pyo3-polars/src/ffi/to_rust.rs @@ -1,6 +1,6 @@ use crate::error::PyPolarsErr; -use polars::export::arrow::ffi; use polars::prelude::*; +use polars_arrow::ffi; use pyo3::ffi::Py_uintptr_t; use pyo3::prelude::*; diff --git a/pyo3-polars/src/lib.rs b/pyo3-polars/src/lib.rs index 281da03..65437fa 100644 --- a/pyo3-polars/src/lib.rs +++ b/pyo3-polars/src/lib.rs @@ -55,9 +55,8 @@ use once_cell::sync::Lazy; use pyo3::prelude::*; pub use types::*; -pub(crate) static POLARS: Lazy = Lazy::new(|| { - Python::with_gil(|py| PyModule::import_bound(py, "polars").unwrap().to_object(py)) -}); +pub(crate) static POLARS: Lazy> = + Lazy::new(|| Python::with_gil(|py| PyModule::import(py, "polars").unwrap().unbind())); -pub(crate) static SERIES: Lazy = +pub(crate) static SERIES: Lazy> = Lazy::new(|| Python::with_gil(|py| POLARS.getattr(py, "Series").unwrap())); diff --git a/pyo3-polars/src/types.rs b/pyo3-polars/src/types.rs index dd4fede..5fe4ff7 100644 --- a/pyo3-polars/src/types.rs +++ b/pyo3-polars/src/types.rs @@ -1,7 +1,10 @@ +use std::convert::Infallible; + use super::*; + use crate::error::PyPolarsErr; use crate::ffi::to_py::to_py_array; -use polars::export::arrow; +use polars_arrow as arrow; use polars_core::datatypes::{CompatLevel, DataType}; use polars_core::prelude::*; use polars_core::utils::materialize_dyn_int; @@ -11,14 +14,16 @@ use polars_lazy::frame::LazyFrame; use polars_plan::dsl::Expr; #[cfg(feature = "lazy")] use polars_plan::plans::DslPlan; +#[cfg(feature = "lazy")] +use polars_utils::pl_serialize; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::ffi::Py_uintptr_t; use pyo3::intern; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; -use pyo3::types::PyDict; #[cfg(feature = "dtype-struct")] use pyo3::types::PyList; +use pyo3::types::{PyBytes, PyDict, PyString}; #[cfg(feature = "dtype-categorical")] pub(crate) fn get_series(obj: &Bound<'_, PyAny>) -> PyResult { @@ -98,14 +103,18 @@ impl<'py> FromPyObject<'py> for PyTimeUnit { } } -impl ToPyObject for PyTimeUnit { - fn to_object(&self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for PyTimeUnit { + type Target = PyString; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { let time_unit = match self.0 { TimeUnit::Nanoseconds => "ns", TimeUnit::Microseconds => "us", TimeUnit::Milliseconds => "ms", }; - time_unit.into_py(py) + time_unit.into_pyobject(py) } } @@ -167,7 +176,7 @@ impl<'a> FromPyObject<'a> for PySeries { let py_name = name.str()?; let name = py_name.to_cow()?; - let kwargs = PyDict::new_bound(ob.py()); + let kwargs = PyDict::new(ob.py()); if let Ok(compat_level) = ob.call_method0("_newest_compat_level") { let compat_level = compat_level.extract().unwrap(); let compat_level = @@ -188,7 +197,7 @@ impl<'a> FromPyObject<'a> for PyDataFrame { let series = ob.call_method0("get_columns")?; let n = ob.getattr("width")?.extract::()?; let mut columns = Vec::with_capacity(n); - for pyseries in series.iter()? { + for pyseries in series.try_iter()? { let pyseries = pyseries?; let s = pyseries.extract::()?.0; columns.push(s.into_column()); @@ -204,12 +213,18 @@ impl<'a> FromPyObject<'a> for PyDataFrame { #[cfg(feature = "lazy")] impl<'a> FromPyObject<'a> for PyLazyFrame { fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult { - let s = ob.call_method0("__getstate__")?.extract::>()?; - let lp: DslPlan = ciborium::de::from_reader(&*s).map_err( + let s = ob.call_method0("__getstate__")?; + let b = s.extract::>()?; + let b = b.as_bytes(); + + let lp: DslPlan = pl_serialize::SerializeOptions::default() + .deserialize_from_reader(&*b) + .map_err( |e| PyPolarsErr::Other( format!("Error when deserializing LazyFrame. This may be due to mismatched polars versions. {}", e) ) )?; + Ok(PyLazyFrame(LazyFrame::from(lp))) } } @@ -218,17 +233,25 @@ impl<'a> FromPyObject<'a> for PyLazyFrame { impl<'a> FromPyObject<'a> for PyExpr { fn extract_bound(ob: &Bound<'a, PyAny>) -> PyResult { let s = ob.call_method0("__getstate__")?.extract::>()?; - let e: Expr = ciborium::de::from_reader(&*s).map_err( + + let e: Expr = pl_serialize::SerializeOptions::default() + .deserialize_from_reader(&*s) + .map_err( |e| PyPolarsErr::Other( format!("Error when deserializing 'Expr'. This may be due to mismatched polars versions. {}", e) ) )?; + Ok(PyExpr(e)) } } -impl IntoPy for PySeries { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for PySeries { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let polars = POLARS.bind(py); let s = SERIES.bind(py); match s @@ -284,64 +307,83 @@ impl IntoPy for PySeries { } } - pyseries.to_object(py) + Ok(pyseries) } // Go via pyarrow Err(_) => { let s = self.0.rechunk(); let name = s.name().as_str(); let arr = s.to_arrow(0, CompatLevel::oldest()); - let pyarrow = py.import_bound("pyarrow").expect("pyarrow not installed"); + let pyarrow = py.import("pyarrow").expect("pyarrow not installed"); - let arg = to_py_array(arr, py, pyarrow).unwrap(); + let arg = to_py_array(arr, pyarrow).unwrap(); let s = polars.call_method1("from_arrow", (arg,)).unwrap(); let s = s.call_method1("rename", (name,)).unwrap(); - s.to_object(py) + Ok(s) } } } } -impl IntoPy for PyDataFrame { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for PyDataFrame { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let pyseries = self .0 .get_columns() .iter() - .map(|s| PySeries(s.as_materialized_series().clone()).into_py(py)) - .collect::>(); + .map(|s| PySeries(s.as_materialized_series().clone()).into_pyobject(py)) + .collect::>>()?; let polars = POLARS.bind(py); - let df_object = polars.call_method1("DataFrame", (pyseries,)).unwrap(); - df_object.into_py(py) + polars.call_method1("DataFrame", (pyseries,)) } } #[cfg(feature = "lazy")] -impl IntoPy for PyLazyFrame { - fn into_py(self, py: Python<'_>) -> PyObject { +impl<'py> IntoPyObject<'py> for PyLazyFrame { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + dbg!("into py"); let polars = POLARS.bind(py); - let cls = polars.getattr("LazyFrame").unwrap(); + let cls = polars.getattr("LazyFrame")?; let instance = cls.call_method1(intern!(py, "__new__"), (&cls,)).unwrap(); - let mut writer: Vec = vec![]; - ciborium::ser::into_writer(&self.0.logical_plan, &mut writer).unwrap(); - instance.call_method1("__setstate__", (&*writer,)).unwrap(); - instance.into_py(py) + let buf = pl_serialize::SerializeOptions::default() + .serialize_to_bytes(&self.0.logical_plan) + .unwrap(); + instance.call_method1("__setstate__", (&buf,))?; + Ok(instance) } } #[cfg(feature = "lazy")] -impl IntoPy for PyExpr { - fn into_py(self, py: Python<'_>) -> PyObject { - let polars = POLARS.bind(py); - let cls = polars.getattr("Expr").unwrap(); - let instance = cls.call_method1(intern!(py, "__new__"), (&cls,)).unwrap(); - let mut writer: Vec = vec![]; - ciborium::ser::into_writer(&self.0, &mut writer).unwrap(); +impl<'py> IntoPyObject<'py> for PyExpr { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; - instance.call_method1("__setstate__", (&*writer,)).unwrap(); - instance.into_py(py) + fn into_pyobject(self, py: Python<'py>) -> Result { + let polars = POLARS.bind(py); + let cls = polars.getattr("Expr")?; + let instance = cls.call_method1(intern!(py, "__new__"), (&cls,))?; + + let buf = pl_serialize::SerializeOptions::default() + .serialize_to_bytes(&self.0) + .unwrap(); + + instance + .call_method1("__setstate__", (&buf,)) + .map_err(|err| { + let msg = format!("deserialization failed: {err}"); + PyValueError::new_err(msg) + }) } } @@ -351,103 +393,109 @@ pub(crate) fn to_series(py: Python, s: PySeries) -> PyObject { let constructor = series .getattr(intern!(series.py(), "_from_pyseries")) .unwrap(); - constructor.call1((s,)).unwrap().into_py(py) + constructor + .call1((s,)) + .unwrap() + .into_pyobject(py) + .unwrap() + .into() } -impl ToPyObject for PyDataType { - fn to_object(&self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for PyDataType { + type Target = PyAny; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { let pl = POLARS.bind(py); match &self.0 { DataType::Int8 => { let class = pl.getattr(intern!(py, "Int8")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Int16 => { let class = pl.getattr(intern!(py, "Int16")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Int32 => { let class = pl.getattr(intern!(py, "Int32")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Int64 => { let class = pl.getattr(intern!(py, "Int64")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::UInt8 => { let class = pl.getattr(intern!(py, "UInt8")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::UInt16 => { let class = pl.getattr(intern!(py, "UInt16")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::UInt32 => { let class = pl.getattr(intern!(py, "UInt32")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::UInt64 => { let class = pl.getattr(intern!(py, "UInt64")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Float32 => { let class = pl.getattr(intern!(py, "Float32")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Float64 | DataType::Unknown(UnknownKind::Float) => { let class = pl.getattr(intern!(py, "Float64")).unwrap(); - class.call0().unwrap().into() + class.call0() } #[cfg(feature = "dtype-decimal")] DataType::Decimal(precision, scale) => { let class = pl.getattr(intern!(py, "Decimal")).unwrap(); let args = (*precision, *scale); - class.call1(args).unwrap().into() + class.call1(args) } DataType::Boolean => { let class = pl.getattr(intern!(py, "Boolean")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::String | DataType::Unknown(UnknownKind::Str) => { let class = pl.getattr(intern!(py, "String")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Binary => { let class = pl.getattr(intern!(py, "Binary")).unwrap(); - class.call0().unwrap().into() + class.call0() } #[cfg(feature = "dtype-array")] DataType::Array(inner, size) => { let class = pl.getattr(intern!(py, "Array")).unwrap(); - let inner = PyDataType(*inner.clone()).to_object(py); + let inner = PyDataType(*inner.clone()).into_pyobject(py)?; let args = (inner, *size); - class.call1(args).unwrap().into() + class.call1(args) } DataType::List(inner) => { let class = pl.getattr(intern!(py, "List")).unwrap(); - let inner = PyDataType(*inner.clone()).to_object(py); - class.call1((inner,)).unwrap().into() + let inner = PyDataType(*inner.clone()).into_pyobject(py)?; + class.call1((inner,)) } DataType::Date => { let class = pl.getattr(intern!(py, "Date")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Datetime(tu, tz) => { let datetime_class = pl.getattr(intern!(py, "Datetime")).unwrap(); - datetime_class - .call1((tu.to_ascii(), tz.as_ref().map(|s| s.as_str()))) - .unwrap() - .into() + datetime_class.call1((tu.to_ascii(), tz.as_ref().map(|s| s.as_str()))) } DataType::Duration(tu) => { let duration_class = pl.getattr(intern!(py, "Duration")).unwrap(); - duration_class.call1((tu.to_ascii(),)).unwrap().into() + duration_class.call1((tu.to_ascii(),)) } #[cfg(feature = "object")] DataType::Object(_, _) => { let class = pl.getattr(intern!(py, "Object")).unwrap(); - class.call0().unwrap().into() + class.call0() } #[cfg(feature = "dtype-categorical")] DataType::Categorical(_, ordering) => { @@ -456,7 +504,7 @@ impl ToPyObject for PyDataType { CategoricalOrdering::Physical => "physical", CategoricalOrdering::Lexical => "lexical", }; - class.call1((ordering,)).unwrap().into() + class.call1((ordering,)) } #[cfg(feature = "dtype-categorical")] DataType::Enum(rev_map, _) => { @@ -465,31 +513,34 @@ impl ToPyObject for PyDataType { let class = pl.getattr(intern!(py, "Enum")).unwrap(); let s = Series::from_arrow("category".into(), categories.clone().boxed()).unwrap(); let series = to_series(py, PySeries(s)); - return class.call1((series,)).unwrap().into(); + return class.call1((series,)); } - DataType::Time => pl.getattr(intern!(py, "Time")).unwrap().into(), + DataType::Time => pl.getattr(intern!(py, "Time")), #[cfg(feature = "dtype-struct")] DataType::Struct(fields) => { let field_class = pl.getattr(intern!(py, "Field")).unwrap(); - let iter = fields.iter().map(|fld| { - let name = fld.name().as_str(); - let dtype = PyDataType(fld.dtype().clone()).to_object(py); - field_class.call1((name, dtype)).unwrap() - }); - let fields = PyList::new_bound(py, iter); + let iter = fields + .iter() + .map(|fld| { + let name = fld.name().as_str(); + let dtype = PyDataType(fld.dtype().clone()).into_pyobject(py)?; + field_class.call1((name, dtype)) + }) + .collect::>>()?; + let fields = PyList::new(py, iter)?; let struct_class = pl.getattr(intern!(py, "Struct")).unwrap(); - struct_class.call1((fields,)).unwrap().into() + struct_class.call1((fields,)) } DataType::Null => { let class = pl.getattr(intern!(py, "Null")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::Unknown(UnknownKind::Int(v)) => { - PyDataType(materialize_dyn_int(*v).dtype()).to_object(py) + PyDataType(materialize_dyn_int(*v).dtype()).into_pyobject(py) } DataType::Unknown(_) => { let class = pl.getattr(intern!(py, "Unknown")).unwrap(); - class.call0().unwrap().into() + class.call0() } DataType::BinaryOffset => { panic!("this type isn't exposed to python") @@ -500,13 +551,17 @@ impl ToPyObject for PyDataType { } } -impl IntoPy for PySchema { - fn into_py(self, py: Python<'_>) -> PyObject { - let dict = PyDict::new_bound(py); +impl<'py> IntoPyObject<'py> for PySchema { + type Target = PyDict; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let dict = PyDict::new(py); for (k, v) in self.0.iter() { - dict.set_item(k.as_str(), PyDataType(v.clone())).unwrap(); + dict.set_item(k.as_str(), PyDataType(v.clone()).into_pyobject(py)?)?; } - dict.into_py(py) + Ok(dict) } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 11c73a2..25282ae 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-11-28" +channel = "nightly-2025-01-05"