diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19e7f6e..e8fdd9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ permissions: contents: read env: - PYTHON_VERSION: "3.9" + PYTHON_VERSION: "3.12" RUST_BACKTRACE: "1" jobs: diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 691c34a..9646758 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,15 +8,16 @@ build: os: "ubuntu-lts-latest" tools: python: "3.13" + rust: "1.96" jobs: post_checkout: - (git --no-pager log --pretty="tformat:%s" -1 | grep -vqF "[skip-rtd]") || exit 183 - git fetch --unshallow || true - pre_create_environment: - # install rust manually. https://github.com/readthedocs/readthedocs.org/issues/11488 - - asdf install rust 1.91 - - asdf global rust 1.91 + #pre_create_environment: + # # install rust manually. https://github.com/readthedocs/readthedocs.org/issues/11488 + # - asdf install rust 1.96 + # - asdf global rust 1.96 python: install: diff --git a/Cargo.toml b/Cargo.toml index 03527e7..9c62987 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,11 +4,13 @@ resolver = "2" members = ["h3ronpy", "crates/h3arrow"] [workspace.dependencies] -geo = "0.29" +geo = "0.33" geo-types = "0.7" -h3o = { version = "0.7" } +h3o = { version = "0.10" } rayon = "^1" -arrow = { version = "53" } +arrow = { version = "58" } +geoarrow-array = { version = "0.8" } +geoarrow = { version = "0.8" } [profile.release] lto = "thin" diff --git a/crates/h3arrow/Cargo.toml b/crates/h3arrow/Cargo.toml index cedaefe..3d9b572 100644 --- a/crates/h3arrow/Cargo.toml +++ b/crates/h3arrow/Cargo.toml @@ -1,35 +1,42 @@ [package] name = "h3arrow" -version = "0.4.0" +version = "0.5.0" edition = "2021" authors = ["Nico Mandery "] description = "Integration of the H3 geospatial grid with the arrow memory model" license = "MIT" keywords = ["geo", "spatial", "h3", "arrow"] readme = "README.md" -homepage = "https://github.com/nmandery/h3arrow" -repository = "https://github.com/nmandery/h3arrow" +homepage = "https://github.com/nmandery/h3ronpy" +repository = "https://github.com/nmandery/h3ronpy" [package.metadata.docs.rs] all-features = true [features] -geoarrow = ["dep:geoarrow", "dep:geozero"] -rayon = ["dep:rayon", "geoarrow/rayon"] +geoarrow = [ + "dep:geoarrow", + "dep:geozero", + "dep:geoarrow-array", + "dep:geo-traits", +] +rayon = ["dep:rayon"] spatial_index = ["dep:rstar"] [dependencies] ahash = "0.8" arrow = { workspace = true } -geoarrow = { git = "https://github.com/geoarrow/geoarrow-rs", rev = "3ecf7dfc1816261b84f813eaf2a0174f2b5752d8", optional = true } +geoarrow = { workspace = true, optional = true } +geoarrow-array = { workspace = true, optional = true } +geo-traits = { version = "0.3", optional = true } geo-types = { workspace = true } geo = { workspace = true } -geozero = { version = "^0.14", default-features = false, features = [ +geozero = { version = "^0.15", default-features = false, features = [ "with-geo", "with-wkb", ], optional = true } h3o = { workspace = true, features = ["geo"] } -nom = "7" +nom = "8" rayon = { workspace = true, optional = true } -rstar = { version = "0.12", optional = true } -thiserror = "1" +rstar = { version = "0.13", optional = true } +thiserror = "2" diff --git a/crates/h3arrow/src/algorithm/change_resolution.rs b/crates/h3arrow/src/algorithm/change_resolution.rs index c03705d..bdfe8e3 100644 --- a/crates/h3arrow/src/algorithm/change_resolution.rs +++ b/crates/h3arrow/src/algorithm/change_resolution.rs @@ -2,7 +2,7 @@ use crate::array::{CellIndexArray, H3ListArray, H3ListArrayBuilder}; use crate::error::Error; use h3o::{CellIndex, Resolution}; use std::cmp::Ordering; -use std::iter::repeat; +use std::iter::repeat_n; pub struct ChangedResolutionPair { /// values before the resolution change @@ -109,7 +109,7 @@ impl ChangeResolutionOp for CellIndexArray { self.iter().flatten().for_each(|cell| { let len_before = after_vec.len(); extend_with_cell(&mut after_vec, cell, resolution); - before_vec.extend(repeat(cell).take(after_vec.len() - len_before)); + before_vec.extend(repeat_n(cell, after_vec.len() - len_before)); }); Ok(ChangedResolutionPair { diff --git a/crates/h3arrow/src/algorithm/compact.rs b/crates/h3arrow/src/algorithm/compact.rs index a64236f..8d932e5 100644 --- a/crates/h3arrow/src/algorithm/compact.rs +++ b/crates/h3arrow/src/algorithm/compact.rs @@ -17,7 +17,10 @@ where impl CompactOp for CellIndexArray { fn compact(&self) -> Result { - Ok(CellIndex::compact(self.iter().flatten())?.collect()) + // TODO: find a way to avoid allocation of a new vec + let mut cells: Vec<_> = self.iter().flatten().collect(); + CellIndex::compact(&mut cells)?; + Ok(cells.into()) } fn compact_mixed_resolutions(&self) -> Result { @@ -76,7 +79,8 @@ impl CellSet { let mut compacted_in = std::mem::take(&mut self.cells_by_resolution[r_idx]); compacted_in.sort_unstable(); compacted_in.dedup(); - for cell in CellIndex::compact(compacted_in.into_iter())? { + CellIndex::compact(&mut compacted_in)?; + for cell in compacted_in { self.insert(cell); } res = h3_res.pred(); diff --git a/crates/h3arrow/src/algorithm/concave_hull.rs b/crates/h3arrow/src/algorithm/concave_hull.rs index 88def66..cc99ea8 100644 --- a/crates/h3arrow/src/algorithm/concave_hull.rs +++ b/crates/h3arrow/src/algorithm/concave_hull.rs @@ -3,29 +3,50 @@ use crate::array::to_geo::{ vertexindexarray_to_multipoint, }; use crate::array::{CellIndexArray, DirectedEdgeIndexArray, VertexIndexArray}; -use geo::concave_hull::ConcaveHull; +use geo::concave_hull::{ConcaveHull, ConcaveHullOptions}; use geo_types::Polygon; impl ConcaveHull for CellIndexArray { type Scalar = f64; - fn concave_hull(&self, concavity: Self::Scalar) -> Polygon { - cellindexarray_to_multipolygon(self).concave_hull(concavity) + fn concave_hull(&self) -> Polygon { + cellindexarray_to_multipolygon(self).concave_hull() + } + + fn concave_hull_with_options( + &self, + concave_hull_options: ConcaveHullOptions, + ) -> Polygon { + cellindexarray_to_multipolygon(self).concave_hull_with_options(concave_hull_options) } } impl ConcaveHull for VertexIndexArray { type Scalar = f64; - fn concave_hull(&self, concavity: Self::Scalar) -> Polygon { - vertexindexarray_to_multipoint(self).concave_hull(concavity) + fn concave_hull(&self) -> Polygon { + vertexindexarray_to_multipoint(self).concave_hull() + } + + fn concave_hull_with_options( + &self, + concave_hull_options: ConcaveHullOptions, + ) -> Polygon { + vertexindexarray_to_multipoint(self).concave_hull_with_options(concave_hull_options) } } impl ConcaveHull for DirectedEdgeIndexArray { type Scalar = f64; - fn concave_hull(&self, concavity: Self::Scalar) -> Polygon { - directededgeindexarray_to_multipoint(self).concave_hull(concavity) + fn concave_hull(&self) -> Polygon { + directededgeindexarray_to_multipoint(self).concave_hull() + } + + fn concave_hull_with_options( + &self, + concave_hull_options: ConcaveHullOptions, + ) -> Polygon { + directededgeindexarray_to_multipoint(self).concave_hull_with_options(concave_hull_options) } } diff --git a/crates/h3arrow/src/algorithm/localij.rs b/crates/h3arrow/src/algorithm/localij.rs index 55a0b53..9ce768b 100644 --- a/crates/h3arrow/src/algorithm/localij.rs +++ b/crates/h3arrow/src/algorithm/localij.rs @@ -2,7 +2,7 @@ use crate::array::{CellIndexArray, H3ArrayBuilder}; use crate::error::Error; use arrow::array::Int32Array; use h3o::{CellIndex, CoordIJ, LocalIJ}; -use std::iter::repeat; +use std::iter::repeat_n; pub struct LocalIJArrays { pub anchors: CellIndexArray, @@ -97,7 +97,7 @@ impl ToLocalIJOp for CellIndexArray { anchor: CellIndex, set_failing_to_invalid: bool, ) -> Result { - let anchors = CellIndexArray::from_iter(repeat(anchor).take(self.len())); + let anchors = CellIndexArray::from_iter(repeat_n(anchor, self.len())); self.to_local_ij_array(anchors, set_failing_to_invalid) } diff --git a/crates/h3arrow/src/algorithm/string.rs b/crates/h3arrow/src/algorithm/string.rs index a065d7b..94c97bc 100644 --- a/crates/h3arrow/src/algorithm/string.rs +++ b/crates/h3arrow/src/algorithm/string.rs @@ -13,7 +13,7 @@ use nom::branch::alt; use nom::bytes::complete::{tag, take_while, take_while_m_n}; use nom::combinator::map_res; use nom::number::complete::double; -use nom::IResult; +use nom::{IResult, Parser}; use crate::error::Error; @@ -67,13 +67,15 @@ fn is_whitespace(c: char) -> bool { } fn seperator(s: &str) -> IResult<&str, &str> { - alt((tag(","), (tag(";"))))(s) + alt((tag(","), (tag(";")))).parse(s) } fn u8_str(s: &str) -> IResult<&str, u8> { - map_res(take_while_m_n(1, 2, |c: char| c.is_ascii_digit()), |u8s| { - u8::from_str(u8s) - })(s) + map_res( + take_while_m_n(1, 2, |c: char| c.is_ascii_digit()), + |u8s: &str| u8::from_str(u8s), + ) + .parse(s) } fn parse_coordinate_and_resolution(s: &str) -> IResult<&str, (Coord, u8)> { diff --git a/crates/h3arrow/src/array/from_geo.rs b/crates/h3arrow/src/array/from_geo.rs index cf0ff9e..4ac2c5e 100644 --- a/crates/h3arrow/src/array/from_geo.rs +++ b/crates/h3arrow/src/array/from_geo.rs @@ -318,11 +318,9 @@ pub fn geometry_to_cells( cells.sort_unstable(); cells.dedup(); - let cells = if options.compact { - CellIndex::compact(cells)?.collect() - } else { - cells - }; + if options.compact { + CellIndex::compact(&mut cells)? + } Ok(cells) } diff --git a/crates/h3arrow/src/array/from_geoarrow.rs b/crates/h3arrow/src/array/from_geoarrow.rs index 1303fd6..60fcc19 100644 --- a/crates/h3arrow/src/array/from_geoarrow.rs +++ b/crates/h3arrow/src/array/from_geoarrow.rs @@ -7,10 +7,10 @@ use crate::array::from_geo::geometry_to_cells; use crate::array::{CellIndexArray, H3ListArray}; use crate::error::Error; use arrow::array::OffsetSizeTrait; +use geo_traits::to_geo::ToGeoGeometry; use geo_types::Geometry; -use geoarrow::array::WKBArray; -use geoarrow::trait_::ArrayAccessor; -use geoarrow::ArrayBase; +use geoarrow::array::{GenericWkbArray, GeometryArray}; +use geoarrow_array::{GeoArrowArray, GeoArrowArrayAccessor}; use h3o::CellIndex; #[cfg(feature = "rayon")] use rayon::prelude::{IntoParallelIterator, ParallelIterator}; @@ -22,17 +22,29 @@ macro_rules! impl_to_cells { &self, options: &ToCellsOptions, ) -> Result, Error> { - self.iter_geo() - .map(|v| v.map(Geometry::from)) - .to_celllistarray(options) + let geom_array: GeometryArray = self.clone().into(); + let geoms: Vec> = geom_array + .iter() + .map(|opt| match opt { + None => Ok(None), + Some(r) => Ok(r.map_err(Error::from)?.try_to_geometry()), + }) + .collect::>()?; + geoms.into_iter().to_celllistarray(options) } } impl ToCellIndexArray for $array_type { fn to_cellindexarray(&self, options: &ToCellsOptions) -> Result { - self.iter_geo() - .map(|v| v.map(Geometry::from)) - .to_cellindexarray(options) + let geom_array: GeometryArray = self.clone().into(); + let geoms: Vec> = geom_array + .iter() + .map(|opt| match opt { + None => Ok(None), + Some(r) => Ok(r.map_err(Error::from)?.try_to_geometry()), + }) + .collect::>()?; + geoms.into_iter().to_cellindexarray(options) } } }; @@ -45,7 +57,7 @@ impl_to_cells!(geoarrow::array::MultiPolygonArray); impl_to_cells!(geoarrow::array::PointArray); impl_to_cells!(geoarrow::array::PolygonArray); -impl ToCellListArray for WKBArray { +impl ToCellListArray for GenericWkbArray { fn to_celllistarray( &self, options: &ToCellsOptions, @@ -57,10 +69,12 @@ impl ToCellListArray for WKBArray { let pos_iter = (0..self.len()).into_par_iter(); let cell_vecs = pos_iter - .map(|pos| { - self.get_as_geo(pos) - .map(|geom| geometry_to_cells(&geom, options)) - .transpose() + .map(|pos| match self.get(pos).map_err(Error::from)? { + None => Ok(None), + Some(wkb) => match wkb.try_to_geometry() { + Some(geom) => geometry_to_cells(&geom, options).map(Some), + None => Ok(None), + }, }) .collect::, _>>()?; @@ -68,7 +82,7 @@ impl ToCellListArray for WKBArray { } } -impl ToCellIndexArray for WKBArray { +impl ToCellIndexArray for GenericWkbArray { fn to_cellindexarray(&self, options: &ToCellsOptions) -> Result { let cellindexarray = self.to_celllistarray(options)?.into_flattened()?; diff --git a/crates/h3arrow/src/array/mod.rs b/crates/h3arrow/src/array/mod.rs index ec52125..c7aa79f 100644 --- a/crates/h3arrow/src/array/mod.rs +++ b/crates/h3arrow/src/array/mod.rs @@ -112,7 +112,7 @@ where /// Returns an iterator over the values and validity as Option. #[allow(clippy::type_complexity)] - pub fn iter(&self) -> PrimitiveArrayH3IndexIter { + pub fn iter(&self) -> PrimitiveArrayH3IndexIter<'_, IX> { // as the array contents have been validated upon construction, we just transmute to the h3o type PrimitiveArrayH3IndexIter { primitive_array_iter: self.primitive_array.iter(), diff --git a/crates/h3arrow/src/array/to_geo.rs b/crates/h3arrow/src/array/to_geo.rs index 7c21294..aa79696 100644 --- a/crates/h3arrow/src/array/to_geo.rs +++ b/crates/h3arrow/src/array/to_geo.rs @@ -5,6 +5,7 @@ use crate::array::{ use crate::error::Error; use geo::{CoordsIter, ToRadians}; use geo_types::{Coord, Line, LineString, MultiPoint, MultiPolygon, Point, Polygon}; +use h3o::geom::SolventBuilder; use h3o::{CellIndex, DirectedEdgeIndex, LatLng, VertexIndex}; use std::convert::Infallible; use std::iter::{repeat, Map, Repeat, Zip}; @@ -202,14 +203,8 @@ impl ToMultiPolygons for H3ListArray { fn to_multipolygons(&self, use_degrees: bool) -> Result { self.iter_arrays() .map(|opt| { - opt.map(|res| { - res.and_then(|array| { - array - .to_multipolygons(use_degrees) - .map_err(Self::Error::from) - }) - }) - .transpose() + opt.map(|res| res.and_then(|array| array.to_multipolygons(use_degrees))) + .transpose() }) .collect() } @@ -220,7 +215,8 @@ impl ToMultiPolygons for CellIndexArray { type Output = MultiPolygon; fn to_multipolygons(&self, use_degrees: bool) -> Result { - let mut multi_polygons = h3o::geom::dissolve(self.iter().flatten())?; + let solvent = SolventBuilder::new().build(); + let mut multi_polygons = solvent.dissolve(self.iter().flatten())?; if !use_degrees { multi_polygons.to_radians_in_place(); } diff --git a/crates/h3arrow/src/array/to_geoarrow.rs b/crates/h3arrow/src/array/to_geoarrow.rs index b3013d2..e454d30 100644 --- a/crates/h3arrow/src/array/to_geoarrow.rs +++ b/crates/h3arrow/src/array/to_geoarrow.rs @@ -2,14 +2,17 @@ use crate::array::to_geo::{ IterLines, IterPoints, IterPolygons, ToLineStrings, ToPoints, ToPolygons, }; use crate::array::{H3Array, H3IndexArrayValue}; +use crate::error::Error; use arrow::array::{Array, OffsetSizeTrait}; use geo::point; use geo_types::LineString; use geoarrow::array::{ - LineStringArray, LineStringBuilder, PointArray, PointBuilder, PolygonArray, PolygonBuilder, - WKBArray, WKBBuilder, WKBCapacity, + GenericWkbArray, LineStringArray, LineStringBuilder, PointArray, PointBuilder, PolygonArray, + PolygonBuilder, WkbBuilder, }; -use geoarrow::datatypes::Dimension; +use geoarrow::datatypes::{Dimension, LineStringType, PointType, PolygonType}; +use geoarrow_array::capacity::WkbCapacity; +use std::convert::Infallible; pub trait ToGeoArrowPolygons { type Error; @@ -31,11 +34,9 @@ where ) -> Result { Ok(PolygonBuilder::from_nullable_polygons( &self.to_polygons(use_degrees)?, - Dimension::XY, - Default::default(), - Default::default(), + PolygonType::new(Dimension::XY, Default::default()), ) - .into()) + .finish()) } } @@ -52,11 +53,9 @@ where fn to_geoarrow_points(&self, use_degrees: bool) -> Result { Ok(PointBuilder::from_nullable_points( self.to_points(use_degrees)?.iter().map(|x| x.as_ref()), - Dimension::XY, - Default::default(), - Default::default(), + PointType::new(Dimension::XY, Default::default()), ) - .into()) + .finish()) } } @@ -79,11 +78,9 @@ where ) -> Result { Ok(LineStringBuilder::from_nullable_line_strings( &self.to_linestrings(use_degrees)?, - Dimension::XY, - Default::default(), - Default::default(), + LineStringType::new(Dimension::XY, Default::default()), ) - .into()) + .finish()) } } @@ -92,20 +89,20 @@ pub trait ToWKBPolygons { fn to_wkb_polygons( &self, use_degrees: bool, - ) -> Result, Self::Error>; + ) -> Result, Self::Error>; } impl ToWKBPolygons for H3Array where - Self: IterPolygons, + Self: IterPolygons, T: H3IndexArrayValue, { - type Error = ::Error; + type Error = Error; fn to_wkb_polygons( &self, use_degrees: bool, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { // just use the first value to estimate the required buffer size. This may be off a bit and require // a re-allocation in case the first element is a pentagon let geometry_wkb_size = if let Some(first_value) = self @@ -113,8 +110,8 @@ where .flat_map(|v| v.transpose().ok().flatten()) .next() { - let mut cap = WKBCapacity::new_empty(); - cap.add_polygon(Some(&first_value)); + let mut cap = WkbCapacity::new_empty(); + cap.add_geometry(Some(&first_value)); cap.buffer_capacity() } else { 0 @@ -127,13 +124,13 @@ where .map(|nb| nb.null_count()) .unwrap_or(0), ); - let mut builder = WKBBuilder::with_capacity(WKBCapacity::new( - num_non_null * geometry_wkb_size, - self.len(), - )); + let mut builder = WkbBuilder::with_capacity( + Default::default(), + WkbCapacity::new(num_non_null * geometry_wkb_size, self.len()), + ); for poly in self.iter_polygons(use_degrees) { - let poly = poly.transpose()?; - builder.push_polygon(poly.as_ref()) + let poly = poly.transpose().unwrap_infallible(); + builder.push_geometry(poly.as_ref())? } Ok(builder.finish()) } @@ -144,28 +141,28 @@ pub trait ToWKBLineStrings { fn to_wkb_linestrings( &self, use_degrees: bool, - ) -> Result, Self::Error>; + ) -> Result, Self::Error>; } impl ToWKBLineStrings for H3Array where - Self: IterLines, + Self: IterLines, T: H3IndexArrayValue, { - type Error = ::Error; + type Error = Error; fn to_wkb_linestrings( &self, use_degrees: bool, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { // just use the first value to estimate the required buffer size. All geometries have the same number of coordinates let geometry_wkb_size = if let Some(first_value) = self .iter_lines(use_degrees) .flat_map(|v| v.transpose().ok().flatten()) .next() { - let mut cap = WKBCapacity::new_empty(); - cap.add_line_string(Some(&LineString::from(first_value))); + let mut cap = WkbCapacity::new_empty(); + cap.add_geometry(Some(&LineString::from(first_value))); cap.buffer_capacity() } else { 0 @@ -179,13 +176,13 @@ where .unwrap_or(0), ); - let mut builder = WKBBuilder::with_capacity(WKBCapacity::new( - num_non_null * geometry_wkb_size, - self.len(), - )); + let mut builder = WkbBuilder::with_capacity( + Default::default(), + WkbCapacity::new(num_non_null * geometry_wkb_size, self.len()), + ); for line in self.iter_lines(use_degrees) { - let linestring = line.transpose()?.map(LineString::from); - builder.push_line_string(linestring.as_ref()) + let linestring = line.transpose().unwrap_infallible().map(LineString::from); + builder.push_geometry(linestring.as_ref())? } Ok(builder.finish()) } @@ -196,20 +193,20 @@ pub trait ToWKBPoints { fn to_wkb_points( &self, use_degrees: bool, - ) -> Result, Self::Error>; + ) -> Result, Self::Error>; } impl ToWKBPoints for H3Array where - Self: IterPoints, + Self: IterPoints, T: H3IndexArrayValue, { - type Error = ::Error; + type Error = Error; fn to_wkb_points( &self, use_degrees: bool, - ) -> Result, Self::Error> { + ) -> Result, Self::Error> { // just use the first value to estimate the required buffer size let geometry_wkb_size = if self .iter_points(use_degrees) @@ -217,9 +214,9 @@ where .next() .is_some() { - let mut cap = WKBCapacity::new_empty(); + let mut cap = WkbCapacity::new_empty(); let point = point! {x:0.0f64, y:0.0f64}; - cap.add_point(Some(&point)); + cap.add_geometry(Some(&point)); cap.buffer_capacity() } else { 0 @@ -232,14 +229,26 @@ where .map(|nb| nb.null_count()) .unwrap_or(0), ); - let mut builder = WKBBuilder::with_capacity(WKBCapacity::new( - geometry_wkb_size * num_non_null, - self.len(), - )); + let mut builder = WkbBuilder::with_capacity( + Default::default(), + WkbCapacity::new(geometry_wkb_size * num_non_null, self.len()), + ); for point in self.iter_points(use_degrees) { - let point = point.transpose()?; - builder.push_point(point.as_ref()) + let point = point.transpose().unwrap_infallible(); + builder.push_geometry(point.as_ref())?; } Ok(builder.finish()) } } + +trait UnwrapInfallible { + type Value; + fn unwrap_infallible(self) -> Self::Value; +} + +impl UnwrapInfallible for Result { + type Value = T; + fn unwrap_infallible(self) -> T { + self.expect("unwrap_infallible called on Ok") + } +} diff --git a/crates/h3arrow/src/error.rs b/crates/h3arrow/src/error.rs index 90de67c..2b15a4d 100644 --- a/crates/h3arrow/src/error.rs +++ b/crates/h3arrow/src/error.rs @@ -53,4 +53,8 @@ pub enum Error { #[error(transparent)] IO(#[from] std::io::Error), + + #[cfg(feature = "geoarrow")] + #[error(transparent)] + GeoArrow(#[from] geoarrow::error::GeoArrowError), } diff --git a/crates/h3arrow/src/spatial_index.rs b/crates/h3arrow/src/spatial_index.rs index 3b78f80..30232e9 100644 --- a/crates/h3arrow/src/spatial_index.rs +++ b/crates/h3arrow/src/spatial_index.rs @@ -109,7 +109,7 @@ where debug_assert_eq!(builder.len(), self.array.len()); let envelope = AABB::from_corners(to_coord(rect.min()), to_coord(rect.max())); - let locator = self.rtree.locate_in_envelope_intersecting(&envelope); + let locator = self.rtree.locate_in_envelope_intersecting(envelope); for located_array_position in locator { if let Some(value) = self.array.get(located_array_position.data) { if !builder.get_bit(located_array_position.data) { diff --git a/h3ronpy/CHANGES.rst b/h3ronpy/CHANGES.rst index ab5c9c3..d4539b2 100644 --- a/h3ronpy/CHANGES.rst +++ b/h3ronpy/CHANGES.rst @@ -9,6 +9,36 @@ adheres to `Semantic Versioning `__. +Unreleased +---------- + +- **Breaking**: The minimum supported Python version is now **3.12**. The + wheels are built against the ``abi3`` stable ABI for Python 3.12 and will + no longer install on Python 3.9 - 3.11. Python 3.13 and 3.14 are supported. +- **Breaking**: ``numpy`` is now required in version **>=2.0**. The extension + is built against the numpy 2 ABI via the Rust ``numpy`` 0.28 crate. numpy 1.x + is no longer supported. +- **Breaking**: ``pyarrow`` (used by the ``pandas`` integration and the test + suite) is now required in version **>=24**. Older releases are built against + the numpy 1.x ABI and cannot be imported under numpy 2. +- **Breaking**: ``Shapely`` (test dependency) is now required in version + **>=2.0** for numpy 2 ABI compatibility and for ``shapely.from_wkb``, which + is used by the raster module. +- Upgrade the native (Rust) stack: ``h3o`` 0.7 -> 0.10, ``rasterh3`` 0.10 -> + 0.12, ``geo`` 0.29 -> 0.33, ``arrow`` 53 -> 58, ``geoarrow`` / ``geoarrow-array`` + -> 0.8, ``pyo3`` 0.22 -> 0.28, ``pyo3-arrow`` 0.5.1 -> 0.17, ``ndarray`` 0.16 + -> 0.17, ``numpy`` (Rust) 0.22 -> 0.28. +- Internal: migrate the ``h3arrow`` crate and the Python bindings to the new + geoarrow 0.8 and pyo3 0.28 APIs. The signatures of the Python-facing API are + unchanged; existing user code does not need to be adjusted beyond the + dependency bumps above. +- The WKB-producing functions (``cells_to_wkb_polygons``, ``cells_to_wkb_points``, + ``vertexes_to_wkb_points``, ``directededges_to_wkb_linestrings``) keep + returning ``large_binary`` arrays (i64 offsets) as in 0.22, and now raise a + Python exception instead of panicking when the WKB serialization fails. +- ``ContainmentMode`` is now hashable and can be used as a dict key or in sets. + + 0.22.0 - 2024-11-26 ------------------- diff --git a/h3ronpy/Cargo.toml b/h3ronpy/Cargo.toml index 6f56c19..817c445 100644 --- a/h3ronpy/Cargo.toml +++ b/h3ronpy/Cargo.toml @@ -21,18 +21,19 @@ geo = { workspace = true } h3arrow = { path = "../crates/h3arrow", features = ["geoarrow", "rayon"] } hashbrown = "0.14" itertools = "0.13" -ndarray = { version = "0.16", features = ["rayon"] } -numpy = "0.22" +ndarray = { version = "0.17", features = ["rayon"] } +numpy = "0.28" ordered-float = ">=2.0.1" -py_geo_interface = { git = "https://github.com/nmandery/py_geo_interface", rev = "36723cdbabc2a7aad1746a8c06db17b4e39ce3b9", features = [ +py_geo_interface = { git = "https://github.com/nmandery/py_geo_interface", rev = "e86808489993a3007100b5ce77a0021129d1d107", features = [ "f64", "wkb", ] } -pyo3 = { version = "^0.22", features = [ +pyo3 = { version = "^0.28", features = [ "extension-module", "abi3", - "abi3-py39", + "abi3-py312", ] } -pyo3-arrow = { version = "0.5.1", default-features = false } -rasterh3 = { version = "0.10", features = ["rayon"] } +pyo3-arrow = { version = "0.17", default-features = false } +rasterh3 = { version = "0.12", features = ["rayon"] } +geoarrow-array = { workspace = true } rayon = { workspace = true } diff --git a/h3ronpy/docs/requirements.txt b/h3ronpy/docs/requirements.txt index 188dbf1..5a27685 100644 --- a/h3ronpy/docs/requirements.txt +++ b/h3ronpy/docs/requirements.txt @@ -1,7 +1,7 @@ sphinx sphinx_rtd_theme>=1.2 sphinx-autodoc-typehints -numpy +numpy>=2 h3>=4 matplotlib rasterio @@ -11,4 +11,5 @@ scipy pandas geopandas polars -pyarrow +shapely>=2 +pyarrow>=24 diff --git a/h3ronpy/docs/source/conf.py b/h3ronpy/docs/source/conf.py index 27150b3..d9b8b69 100644 --- a/h3ronpy/docs/source/conf.py +++ b/h3ronpy/docs/source/conf.py @@ -8,7 +8,7 @@ # -- Project information project = "h3ronpy" -copyright = "2023, the h3ronpy authors" +copyright = "2023-2026, the h3ronpy authors" author = "Nico Mandery" diff --git a/h3ronpy/docs/source/installation.rst b/h3ronpy/docs/source/installation.rst index 3867a49..89e2b08 100644 --- a/h3ronpy/docs/source/installation.rst +++ b/h3ronpy/docs/source/installation.rst @@ -7,6 +7,20 @@ Installation `geopandas` and `polars`. These packages need to be installed separately. +Requirements +------------ + +`h3ronpy` requires **Python 3.12 or newer** (3.12, 3.13 and 3.14 are tested) and **numpy 2.0 or newer**. +The native extension is built against the numpy 2 ABI and the CPython 3.12 stable ABI (``abi3``), so it will not +install or import on older Python or numpy 1.x releases. + +When using the optional integrations, mind their version requirements as well: + +- the ``pandas`` / ``geopandas`` integration requires **pyarrow >= 24** (older releases are built against the numpy 1.x + ABI and cannot be imported under numpy 2) +- the test suite additionally requires **Shapely >= 2.0** + + From PyPi --------- diff --git a/h3ronpy/pyproject.toml b/h3ronpy/pyproject.toml index 76e7e87..69d89ae 100644 --- a/h3ronpy/pyproject.toml +++ b/h3ronpy/pyproject.toml @@ -8,7 +8,7 @@ addopts = "--doctest-modules -v -s" testpaths = ["tests"] [tool.ruff] -target-version = "py39" +target-version = "py312" builtins = ["ellipsis"] exclude = [ ".git", @@ -44,36 +44,42 @@ ban-relative-imports = "all" [tool.black] line-length = 120 -target-version = ["py39"] +target-version = ["py312"] [project] name = "h3ronpy" readme = "../README.rst" license = "MIT" dependencies = [ - "numpy>=1.24", + # numpy 2.0; the Rust `numpy` 0.28 crate and all optional extras below + # are compatible with the numpy 2 ABI. + "numpy>=2.0", "arro3-core>=0.4", ] classifiers = [ "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering :: GIS", ] -requires-python = ">=3.9" +requires-python = ">=3.12" dynamic = ["version"] [project.optional-dependencies] polars = ["polars>=1"] pandas = [ "geopandas>=1", - "pyarrow>=15", + "pyarrow>=24", ] test = [ "rasterio>=1.4", - "Shapely>=1.7", + # Shapely 2 is required for numpy 2 ABI support and for `shapely.from_wkb` + "Shapely>=2.0", "pytest>=6", "h3>=4", "pytest-benchmark", - "pyarrow>=15", + "pyarrow>=24", ] [tool.maturin] diff --git a/h3ronpy/python/h3ronpy/__init__.py b/h3ronpy/python/h3ronpy/__init__.py index 3e3de81..fbffb8c 100644 --- a/h3ronpy/python/h3ronpy/__init__.py +++ b/h3ronpy/python/h3ronpy/__init__.py @@ -16,7 +16,6 @@ if TYPE_CHECKING: import polars as pl - __version__ = version() H3_CRS = "EPSG:4326" diff --git a/h3ronpy/python/h3ronpy/pandas/raster.py b/h3ronpy/python/h3ronpy/pandas/raster.py index 361dd23..3d94a0b 100644 --- a/h3ronpy/python/h3ronpy/pandas/raster.py +++ b/h3ronpy/python/h3ronpy/pandas/raster.py @@ -1,5 +1,3 @@ -import typing - import geopandas as gpd import numpy as np import pandas as pd @@ -21,7 +19,7 @@ def raster_to_dataframe( axis_order: str = "yx", compact: bool = True, geo: bool = False, -) -> typing.Union[gpd.GeoDataFrame, pd.DataFrame]: +) -> gpd.GeoDataFrame | pd.DataFrame: """ Convert a raster/array to a pandas `DataFrame` containing H3 indexes diff --git a/h3ronpy/python/h3ronpy/pandas/vector.py b/h3ronpy/python/h3ronpy/pandas/vector.py index 119996b..98558c3 100644 --- a/h3ronpy/python/h3ronpy/pandas/vector.py +++ b/h3ronpy/python/h3ronpy/pandas/vector.py @@ -1,5 +1,4 @@ from functools import wraps -from typing import Optional import geopandas as gpd import pandas as pd @@ -10,7 +9,7 @@ from h3ronpy import DEFAULT_CELL_COLUMN_NAME, H3_CRS, ContainmentMode -def _geoseries_from_wkb(func, doc: Optional[str] = None, name: Optional[str] = None): +def _geoseries_from_wkb(func, doc: str | None = None, name: str | None = None): @wraps(func) def wrapper(arr, *args, **kw): result = func(arr.values, *args, **kw) diff --git a/h3ronpy/python/h3ronpy/raster.py b/h3ronpy/python/h3ronpy/raster.py index 5dc3d7f..277edc5 100644 --- a/h3ronpy/python/h3ronpy/raster.py +++ b/h3ronpy/python/h3ronpy/raster.py @@ -28,8 +28,6 @@ """ -import typing - import numpy as np import pyarrow as pa @@ -138,7 +136,7 @@ def raster_to_dataframe( def rasterize_cells( - cells, values, size: typing.Union[int, tuple[int, int]], nodata_value=0 + cells, values, size: int | tuple[int, int], nodata_value=0 ) -> tuple[np.ndarray, tuple[float, float, float, float, float, float]]: """ Generate a raster numpy array from arrays of cells and values. diff --git a/h3ronpy/python/h3ronpy/vector.py b/h3ronpy/python/h3ronpy/vector.py index decdb8a..b5f2fe2 100644 --- a/h3ronpy/python/h3ronpy/vector.py +++ b/h3ronpy/python/h3ronpy/vector.py @@ -1,5 +1,3 @@ -from typing import Optional - from arro3.core import Array, DataType, RecordBatch from h3ronpy import ContainmentMode, _to_arrow_array, _to_uint64_array @@ -35,7 +33,7 @@ def coordinates_to_cells(latarray, lngarray, resarray, radians: bool = False) -> ) -def cells_bounds(arr) -> Optional[tuple]: +def cells_bounds(arr) -> tuple | None: """ Bounds of the complete array as a tuple `(minx, miny, maxx, maxy)`. """ diff --git a/h3ronpy/src/array.rs b/h3ronpy/src/array.rs index 4aa5b8f..4a2a026 100644 --- a/h3ronpy/src/array.rs +++ b/h3ronpy/src/array.rs @@ -52,9 +52,10 @@ impl AsRef for PyCellArray { } } -impl<'py> FromPyObject<'py> for PyCellArray { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Ok(Self(pyarray_to_cellindexarray(ob)?)) +impl<'a, 'py> FromPyObject<'a, 'py> for PyCellArray { + type Error = PyErr; + fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(Self(pyarray_to_cellindexarray(&ob)?)) } } @@ -97,9 +98,10 @@ impl AsRef for PyDirectedEdgeArray { } } -impl<'py> FromPyObject<'py> for PyDirectedEdgeArray { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Ok(Self(pyarray_to_directededgeindexarray(ob)?)) +impl<'a, 'py> FromPyObject<'a, 'py> for PyDirectedEdgeArray { + type Error = PyErr; + fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(Self(pyarray_to_directededgeindexarray(&ob)?)) } } @@ -138,8 +140,9 @@ impl AsRef for PyVertexArray { } } -impl<'py> FromPyObject<'py> for PyVertexArray { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { - Ok(Self(pyarray_to_vertexindexarray(ob)?)) +impl<'a, 'py> FromPyObject<'a, 'py> for PyVertexArray { + type Error = PyErr; + fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { + Ok(Self(pyarray_to_vertexindexarray(&ob)?)) } } diff --git a/h3ronpy/src/arrow_interop.rs b/h3ronpy/src/arrow_interop.rs index f2bcb11..7d27f3f 100644 --- a/h3ronpy/src/arrow_interop.rs +++ b/h3ronpy/src/arrow_interop.rs @@ -13,12 +13,12 @@ use pyo3::Python; use crate::error::{IntoPyErr, IntoPyResult}; #[inline] -pub fn h3array_to_pyarray(h3array: H3Array, py: Python) -> PyResult +pub fn h3array_to_pyarray(h3array: H3Array, py: Python<'_>) -> PyResult> where IX: H3IndexArrayValue, { let pa: UInt64Array = h3array.into(); - PyArray::from_array_ref(Arc::new(pa)).to_arro3(py) + PyArray::from_array_ref(Arc::new(pa)).into_arro3(py) } pub(crate) fn pyarray_to_native(obj: &Bound) -> PyResult { diff --git a/h3ronpy/src/error.rs b/h3ronpy/src/error.rs index 2bcecf5..235dff8 100644 --- a/h3ronpy/src/error.rs +++ b/h3ronpy/src/error.rs @@ -1,4 +1,5 @@ use h3arrow::error::Error as A3Error; +use h3arrow::export::geoarrow; use pyo3::exceptions::{PyIOError, PyRuntimeError, PyValueError}; use pyo3::{PyErr, PyResult}; use rasterh3::Error; @@ -17,6 +18,12 @@ impl IntoPyErr for h3arrow::export::arrow::error::ArrowError { } } +impl IntoPyErr for geoarrow::error::GeoArrowError { + fn into_pyerr(self) -> PyErr { + PyRuntimeError::new_err(self.to_string()) + } +} + impl IntoPyErr for A3Error { fn into_pyerr(self) -> PyErr { match self { @@ -38,6 +45,7 @@ impl IntoPyErr for A3Error { | A3Error::LengthMismatch | A3Error::InvalidWKB => PyValueError::new_err(self.to_string()), A3Error::IO(e) => e.into_pyerr(), + A3Error::GeoArrow(e) => e.into_pyerr(), } } } diff --git a/h3ronpy/src/lib.rs b/h3ronpy/src/lib.rs index 539ccbf..1c2082e 100644 --- a/h3ronpy/src/lib.rs +++ b/h3ronpy/src/lib.rs @@ -49,15 +49,15 @@ fn h3ronpyrs(py: Python<'_>, m: &Bound) -> PyResult<()> { m.add_function(wrap_pyfunction!(version, m)?)?; m.add_function(wrap_pyfunction!(is_release_build, m)?)?; - let raster_submod = PyModule::new_bound(py, "raster")?; + let raster_submod = PyModule::new(py, "raster")?; init_raster_submodule(&raster_submod)?; m.add_submodule(&raster_submod)?; - let op_submod = PyModule::new_bound(py, "op")?; + let op_submod = PyModule::new(py, "op")?; init_op_submodule(&op_submod)?; m.add_submodule(&op_submod)?; - let vector_submod = PyModule::new_bound(py, "vector")?; + let vector_submod = PyModule::new(py, "vector")?; init_vector_submodule(&vector_submod)?; m.add_submodule(&vector_submod)?; diff --git a/h3ronpy/src/op/compact.rs b/h3ronpy/src/op/compact.rs index a77acc9..d450913 100644 --- a/h3ronpy/src/op/compact.rs +++ b/h3ronpy/src/op/compact.rs @@ -12,10 +12,10 @@ pub(crate) fn compact( py: Python<'_>, cellarray: PyCellArray, mixed_resolutions: bool, -) -> PyResult { +) -> PyResult> { let cellindexarray = cellarray.into_inner(); let compacted = py - .allow_threads(|| { + .detach(|| { if mixed_resolutions { cellindexarray.compact_mixed_resolutions() } else { @@ -33,9 +33,9 @@ pub(crate) fn uncompact( py: Python<'_>, cellarray: PyCellArray, target_resolution: u8, -) -> PyResult { +) -> PyResult> { let target_resolution = Resolution::try_from(target_resolution).into_pyresult()?; let cellarray = cellarray.into_inner(); - let out = py.allow_threads(|| cellarray.uncompact(target_resolution)); + let out = py.detach(|| cellarray.uncompact(target_resolution)); h3array_to_pyarray(out, py) } diff --git a/h3ronpy/src/op/localij.rs b/h3ronpy/src/op/localij.rs index 03689e0..55f6a6d 100644 --- a/h3ronpy/src/op/localij.rs +++ b/h3ronpy/src/op/localij.rs @@ -8,20 +8,19 @@ use h3arrow::array::CellIndexArray; use h3arrow::h3o::CellIndex; use pyo3::exceptions::PyValueError; use pyo3::prelude::PyAnyMethods; -use pyo3::{pyfunction, Bound, PyAny, PyObject, PyResult, Python}; +use pyo3::{pyfunction, Bound, PyAny, PyResult, Python}; use pyo3_arrow::error::PyArrowResult; use pyo3_arrow::PyRecordBatch; -use std::iter::repeat; use std::sync::Arc; #[pyfunction] #[pyo3(signature = (cellarray, anchor, set_failing_to_invalid = false))] -pub(crate) fn cells_to_localij( - py: Python, +pub(crate) fn cells_to_localij<'py>( + py: Python<'py>, cellarray: PyCellArray, - anchor: &Bound, + anchor: &Bound<'py, PyAny>, set_failing_to_invalid: bool, -) -> PyArrowResult { +) -> PyArrowResult> { let cellindexarray = cellarray.into_inner(); let anchorarray = get_anchor_array(anchor, cellindexarray.len())?; @@ -44,23 +43,23 @@ pub(crate) fn cells_to_localij( Arc::new(anchor), ]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } #[pyfunction] #[pyo3(signature = (anchor, i_array, j_array, set_failing_to_invalid = false))] -pub(crate) fn localij_to_cells( - py: Python<'_>, - anchor: &Bound, - i_array: &Bound, - j_array: &Bound, +pub(crate) fn localij_to_cells<'py>( + py: Python<'py>, + anchor: &Bound<'py, PyAny>, + i_array: &Bound<'py, PyAny>, + j_array: &Bound<'py, PyAny>, set_failing_to_invalid: bool, -) -> PyResult { +) -> PyResult> { let i_array = pyarray_to_native::(i_array)?; let j_array = pyarray_to_native::(j_array)?; let anchorarray = get_anchor_array(anchor, i_array.len())?; - let cellarray = py.allow_threads(|| { + let cellarray = py.detach(|| { let localij_arrays = LocalIJArrays::try_new(anchorarray, i_array, j_array).into_pyresult()?; @@ -77,13 +76,16 @@ pub(crate) fn localij_to_cells( fn get_anchor_array(anchor: &Bound, len: usize) -> PyResult { if let Ok(anchor) = anchor.extract::() { let anchor_cell = CellIndex::try_from(anchor).into_pyresult()?; - Ok(CellIndexArray::from_iter(repeat(anchor_cell).take(len))) + Ok(CellIndexArray::from_iter(std::iter::repeat_n( + anchor_cell, + len, + ))) } else if let Ok(anchorarray) = pyarray_to_cellindexarray(anchor) { Ok(anchorarray) } else { - return Err(PyValueError::new_err(format!( + Err(PyValueError::new_err(format!( "Expected a single cell or an array of cells, found type {:?}", anchor.get_type(), - ))); + ))) } } diff --git a/h3ronpy/src/op/measure.rs b/h3ronpy/src/op/measure.rs index 26a3616..7e8d566 100644 --- a/h3ronpy/src/op/measure.rs +++ b/h3ronpy/src/op/measure.rs @@ -6,21 +6,24 @@ use pyo3_arrow::PyArray; #[pyfunction] #[pyo3(signature = (cellarray))] -pub(crate) fn cells_area_m2(py: Python, cellarray: PyCellArray) -> PyResult { +pub(crate) fn cells_area_m2(py: Python<'_>, cellarray: PyCellArray) -> PyResult> { let out = cellarray.as_ref().area_m2(); - PyArray::from_array_ref(Arc::new(out)).to_arro3(py) + PyArray::from_array_ref(Arc::new(out)).into_arro3(py) } #[pyfunction] #[pyo3(signature = (cellarray))] -pub(crate) fn cells_area_km2(py: Python, cellarray: PyCellArray) -> PyResult { +pub(crate) fn cells_area_km2(py: Python<'_>, cellarray: PyCellArray) -> PyResult> { let out = cellarray.as_ref().area_km2(); - PyArray::from_array_ref(Arc::new(out)).to_arro3(py) + PyArray::from_array_ref(Arc::new(out)).into_arro3(py) } #[pyfunction] #[pyo3(signature = (cellarray))] -pub(crate) fn cells_area_rads2(py: Python, cellarray: PyCellArray) -> PyResult { +pub(crate) fn cells_area_rads2( + py: Python<'_>, + cellarray: PyCellArray, +) -> PyResult> { let out = cellarray.as_ref().area_rads2(); - PyArray::from_array_ref(Arc::new(out)).to_arro3(py) + PyArray::from_array_ref(Arc::new(out)).into_arro3(py) } diff --git a/h3ronpy/src/op/neighbor.rs b/h3ronpy/src/op/neighbor.rs index d0e883e..72bc259 100644 --- a/h3ronpy/src/op/neighbor.rs +++ b/h3ronpy/src/op/neighbor.rs @@ -1,47 +1,46 @@ +use crate::array::PyCellArray; +use crate::arrow_interop::*; +use crate::error::IntoPyResult; +use crate::DEFAULT_CELL_COLUMN_NAME; use arrow::array::{ Array, ArrayRef, GenericListArray, LargeListArray, PrimitiveArray, RecordBatch, UInt32Array, }; use arrow::datatypes::{Field, Schema}; use h3arrow::algorithm::{GridDiskDistances, GridOp, KAggregationMethod}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::{PyObject, PyResult}; +use pyo3::prelude::*; +use pyo3::PyResult; use pyo3_arrow::error::PyArrowResult; use pyo3_arrow::{PyArray, PyRecordBatch}; use std::str::FromStr; use std::sync::Arc; -use crate::array::PyCellArray; -use crate::arrow_interop::*; -use crate::error::IntoPyResult; -use crate::DEFAULT_CELL_COLUMN_NAME; -use pyo3::prelude::*; - #[pyfunction] #[pyo3(signature = (cellarray, k, flatten = false))] pub(crate) fn grid_disk( - py: Python, + py: Python<'_>, cellarray: PyCellArray, k: u32, flatten: bool, -) -> PyResult { +) -> PyResult> { let cellindexarray = cellarray.into_inner(); let listarray = cellindexarray.grid_disk(k).into_pyresult()?; if flatten { let cellindexarray = listarray.into_flattened().into_pyresult()?; h3array_to_pyarray(cellindexarray, py) } else { - PyArray::from_array_ref(Arc::new(LargeListArray::from(listarray))).to_arro3(py) + PyArray::from_array_ref(Arc::new(LargeListArray::from(listarray))).into_arro3(py) } } #[pyfunction] #[pyo3(signature = (cellarray, k, flatten = false))] pub(crate) fn grid_disk_distances( - py: Python, + py: Python<'_>, cellarray: PyCellArray, k: u32, flatten: bool, -) -> PyArrowResult { +) -> PyArrowResult> { let griddiskdistances = cellarray .into_inner() .grid_disk_distances(k) @@ -53,12 +52,12 @@ pub(crate) fn grid_disk_distances( #[pyfunction] #[pyo3(signature = (cellarray, k_min, k_max, flatten = false))] pub(crate) fn grid_ring_distances( - py: Python, + py: Python<'_>, cellarray: PyCellArray, k_min: u32, k_max: u32, flatten: bool, -) -> PyArrowResult { +) -> PyArrowResult> { if k_min >= k_max { return Err(PyValueError::new_err("k_min must be less than k_max").into()); } @@ -71,10 +70,10 @@ pub(crate) fn grid_ring_distances( } fn return_griddiskdistances_table( - py: Python, + py: Python<'_>, griddiskdistances: GridDiskDistances, flatten: bool, -) -> PyArrowResult { +) -> PyArrowResult> { let (cells, distances): (ArrayRef, ArrayRef) = if flatten { ( Arc::new(PrimitiveArray::from( @@ -103,7 +102,7 @@ fn return_griddiskdistances_table( ]); let columns = vec![cells, distances]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } struct KAggregationMethodWrapper(KAggregationMethod); @@ -122,12 +121,12 @@ impl FromStr for KAggregationMethodWrapper { #[pyfunction] #[pyo3(signature = (cellarray, k, aggregation_method))] -pub(crate) fn grid_disk_aggregate_k( - py: Python, +pub(crate) fn grid_disk_aggregate_k<'py>( + py: Python<'py>, cellarray: PyCellArray, k: u32, aggregation_method: &str, -) -> PyArrowResult { +) -> PyArrowResult> { let aggregation_method = KAggregationMethodWrapper::from_str(aggregation_method)?; let griddiskaggk = cellarray @@ -148,5 +147,5 @@ pub(crate) fn grid_disk_aggregate_k( Arc::new(griddiskaggk.distances), ]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } diff --git a/h3ronpy/src/op/resolution.rs b/h3ronpy/src/op/resolution.rs index 25b25fc..cc1eeac 100644 --- a/h3ronpy/src/op/resolution.rs +++ b/h3ronpy/src/op/resolution.rs @@ -18,10 +18,10 @@ pub(crate) fn change_resolution( py: Python<'_>, cellarray: PyCellArray, h3_resolution: u8, -) -> PyResult { +) -> PyResult> { let cellindexarray = cellarray.into_inner(); let h3_resolution = Resolution::try_from(h3_resolution).into_pyresult()?; - let out = py.allow_threads(|| { + let out = py.detach(|| { cellindexarray .change_resolution(h3_resolution) .into_pyresult() @@ -32,25 +32,25 @@ pub(crate) fn change_resolution( #[pyfunction] pub(crate) fn change_resolution_list( - py: Python, + py: Python<'_>, cellarray: PyCellArray, h3_resolution: u8, -) -> PyResult { +) -> PyResult> { let cellindexarray = cellarray.into_inner(); let h3_resolution = Resolution::try_from(h3_resolution).into_pyresult()?; let listarray = cellindexarray .change_resolution_list(h3_resolution) .into_pyresult()?; - PyArray::from_array_ref(Arc::new(LargeListArray::from(listarray))).to_arro3(py) + PyArray::from_array_ref(Arc::new(LargeListArray::from(listarray))).into_arro3(py) } #[pyfunction] pub(crate) fn change_resolution_paired( - py: Python, + py: Python<'_>, cellarray: PyCellArray, h3_resolution: u8, -) -> PyArrowResult { +) -> PyArrowResult> { let cellindexarray = cellarray.into_inner(); let h3_resolution = Resolution::try_from(h3_resolution).into_pyresult()?; let pair = cellindexarray @@ -77,11 +77,14 @@ pub(crate) fn change_resolution_paired( Arc::new(after.primitive_array().clone()), ]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } #[pyfunction] -pub(crate) fn cells_resolution(py: Python, cellarray: PyCellArray) -> PyResult { +pub(crate) fn cells_resolution( + py: Python<'_>, + cellarray: PyCellArray, +) -> PyResult> { let resarray = cellarray.as_ref().resolution(); - PyArray::from_array_ref(Arc::new(resarray.into_inner())).to_arro3(py) + PyArray::from_array_ref(Arc::new(resarray.into_inner())).into_arro3(py) } diff --git a/h3ronpy/src/op/string.rs b/h3ronpy/src/op/string.rs index 620bac7..6bfeb47 100644 --- a/h3ronpy/src/op/string.rs +++ b/h3ronpy/src/op/string.rs @@ -17,9 +17,9 @@ pub(crate) fn cells_parse( py: Python<'_>, stringarray: PyArray, set_failing_to_invalid: bool, -) -> PyResult { +) -> PyResult> { let (boxed_array, _field) = stringarray.into_inner(); - let cells = py.allow_threads(|| { + let cells = py.detach(|| { if let Some(stringarray) = boxed_array.as_any().downcast_ref::() { CellIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid) .into_pyresult() @@ -42,9 +42,9 @@ pub(crate) fn vertexes_parse( py: Python<'_>, stringarray: PyArray, set_failing_to_invalid: bool, -) -> PyResult { +) -> PyResult> { let (boxed_array, _field) = stringarray.into_inner(); - let vertexes = py.allow_threads(|| { + let vertexes = py.detach(|| { if let Some(utf8array) = boxed_array.as_any().downcast_ref::() { VertexIndexArray::parse_genericstringarray(utf8array, set_failing_to_invalid) .into_pyresult() @@ -67,9 +67,9 @@ pub(crate) fn directededges_parse( py: Python<'_>, stringarray: PyArray, set_failing_to_invalid: bool, -) -> PyResult { +) -> PyResult> { let (boxed_array, _field) = stringarray.into_inner(); - let edges = py.allow_threads(|| { + let edges = py.detach(|| { if let Some(stringarray) = boxed_array.as_any().downcast_ref::() { DirectedEdgeIndexArray::parse_genericstringarray(stringarray, set_failing_to_invalid) .into_pyresult() @@ -88,29 +88,35 @@ pub(crate) fn directededges_parse( #[pyfunction] #[pyo3(signature = (cellarray))] -pub(crate) fn cells_to_string(py: Python, cellarray: PyCellArray) -> PyResult { +pub(crate) fn cells_to_string( + py: Python<'_>, + cellarray: PyCellArray, +) -> PyResult> { let stringarray: LargeStringArray = cellarray.as_ref().to_genericstringarray().into_pyresult()?; - PyArray::from_array_ref(Arc::new(stringarray)).to_arro3(py) + PyArray::from_array_ref(Arc::new(stringarray)).into_arro3(py) } #[pyfunction] #[pyo3(signature = (vertexarray))] -pub(crate) fn vertexes_to_string(py: Python, vertexarray: &Bound) -> PyResult { +pub(crate) fn vertexes_to_string<'py>( + py: Python<'py>, + vertexarray: &Bound<'py, PyAny>, +) -> PyResult> { let stringarray: LargeStringArray = pyarray_to_vertexindexarray(vertexarray)? .to_genericstringarray() .into_pyresult()?; - PyArray::from_array_ref(Arc::new(stringarray)).to_arro3(py) + PyArray::from_array_ref(Arc::new(stringarray)).into_arro3(py) } #[pyfunction] #[pyo3(signature = (directededgearray))] -pub(crate) fn directededges_to_string( - py: Python, - directededgearray: &Bound, -) -> PyResult { +pub(crate) fn directededges_to_string<'py>( + py: Python<'py>, + directededgearray: &Bound<'py, PyAny>, +) -> PyResult> { let stringarray: LargeStringArray = pyarray_to_directededgeindexarray(directededgearray)? .to_genericstringarray() .into_pyresult()?; - PyArray::from_array_ref(Arc::new(stringarray)).to_arro3(py) + PyArray::from_array_ref(Arc::new(stringarray)).into_arro3(py) } diff --git a/h3ronpy/src/op/valid.rs b/h3ronpy/src/op/valid.rs index 8126958..5873554 100644 --- a/h3ronpy/src/op/valid.rs +++ b/h3ronpy/src/op/valid.rs @@ -10,12 +10,16 @@ use pyo3_arrow::PyArray; use crate::arrow_interop::*; -fn h3index_valid(py: Python, arr: &Bound, booleanarray: bool) -> PyResult +fn h3index_valid<'py, IX>( + py: Python<'py>, + arr: &Bound<'py, PyAny>, + booleanarray: bool, +) -> PyResult> where IX: H3IndexArrayValue + Send, { let u64array = pyarray_to_uint64array(arr)?; - let validated = py.allow_threads(|| H3Array::::from_iter_with_validity(u64array.iter())); + let validated = py.detach(|| H3Array::::from_iter_with_validity(u64array.iter())); if booleanarray { let nullbuffer = validated @@ -24,7 +28,7 @@ where .cloned() .unwrap_or_else(|| NullBuffer::new_valid(validated.len())); let bools = BooleanArray::from(nullbuffer.into_inner()); - PyArray::from_array_ref(Arc::new(bools)).to_arro3(py) + PyArray::from_array_ref(Arc::new(bools)).into_arro3(py) } else { h3array_to_pyarray(validated, py) } @@ -34,11 +38,11 @@ macro_rules! impl_h3index_valid { ($name:ident, $arr_type:ty) => { #[pyfunction] #[pyo3(signature = (array, booleanarray = false))] - pub(crate) fn $name( - py: Python, - array: &Bound, + pub(crate) fn $name<'py>( + py: Python<'py>, + array: &Bound<'py, PyAny>, booleanarray: bool, - ) -> PyResult { + ) -> PyResult> { h3index_valid::<$arr_type>(py, array, booleanarray) } }; diff --git a/h3ronpy/src/raster.rs b/h3ronpy/src/raster.rs index d4eca1e..b63510f 100644 --- a/h3ronpy/src/raster.rs +++ b/h3ronpy/src/raster.rs @@ -1,7 +1,6 @@ use geo_types::Point; use pyo3_arrow::PyArray; use std::hash::Hash; -use std::iter::repeat; use std::str::FromStr; use std::sync::Arc; @@ -134,7 +133,7 @@ where } else { cells.extend(cell_coverage.into_uncompacted_iter(h3_resolution)); }; - values.extend(repeat(*value).take(cells.len() - len_before)); + values.extend(std::iter::repeat_n(*value, cells.len() - len_before)); } Ok((values, cells)) } @@ -143,17 +142,17 @@ macro_rules! make_raster_to_h3_variant { ($name:ident, $dtype:ty, $array_dtype:ty) => { #[pyfunction] #[pyo3(signature = (np_array, transform, h3_resolution, axis_order_str, compact, nodata_value=None))] - fn $name( - py: Python, + fn $name<'py>( + py: Python<'py>, np_array: PyReadonlyArray2<$dtype>, transform: &Transform, h3_resolution: u8, axis_order_str: &str, compact: bool, nodata_value: Option<$dtype>, - ) -> PyResult<(PyObject, PyObject)> { + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let arr = np_array.as_array(); - let (values, cells) = py.allow_threads(|| raster_to_h3( + let (values, cells) = py.detach(|| raster_to_h3( &arr, transform, &nodata_value, @@ -162,7 +161,7 @@ macro_rules! make_raster_to_h3_variant { compact, ).map(|(values, cells)| (<$array_dtype>::from(values), cells)))?; - let values = PyArray::from_array_ref(Arc::new(values)).to_arro3(py)?; + let values = PyArray::from_array_ref(Arc::new(values)).into_arro3(py)?; let cells = h3array_to_pyarray(CellIndexArray::from(cells), py)?; Ok((values, cells)) @@ -174,20 +173,20 @@ macro_rules! make_raster_to_h3_float_variant { ($name:ident, $dtype:ty, $array_dtype:ty) => { #[pyfunction] #[pyo3(signature = (np_array, transform, h3_resolution, axis_order_str, compact, nodata_value=None))] - fn $name( - py: Python, + fn $name<'py>( + py: Python<'py>, np_array: PyReadonlyArray2<$dtype>, transform: &Transform, h3_resolution: u8, axis_order_str: &str, compact: bool, nodata_value: Option<$dtype>, - ) -> PyResult<(PyObject, PyObject)> { + ) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { let arr = np_array.as_array(); // create a copy with the values wrapped in ordered floats to // support the internal hashing let of_arr = arr.map(|v| OrderedFloat::from(*v)); - let (values, cells) = py.allow_threads(|| raster_to_h3( + let (values, cells) = py.detach(|| raster_to_h3( &of_arr.view(), transform, &nodata_value.map(OrderedFloat::from), @@ -198,7 +197,7 @@ macro_rules! make_raster_to_h3_float_variant { <$array_dtype>::from(values.into_iter().map(|v| v.into_inner()).collect::>()), cells)))?; - let values = PyArray::from_array_ref(Arc::new(values)).to_arro3(py)?; + let values = PyArray::from_array_ref(Arc::new(values)).into_arro3(py)?; let cells = h3array_to_pyarray(CellIndexArray::from(cells), py)?; Ok((values, cells)) @@ -219,7 +218,7 @@ make_raster_to_h3_float_variant!(raster_to_h3_f32, f32, Float32Array); make_raster_to_h3_float_variant!(raster_to_h3_f64, f64, Float64Array); pub fn init_raster_submodule(m: &Bound) -> PyResult<()> { - m.add("Transform", m.py().get_type_bound::())?; + m.add("Transform", m.py().get_type::())?; m.add_function(wrap_pyfunction!(nearest_h3_resolution, m)?)?; m.add_function(wrap_pyfunction!(raster_to_h3_u8, m)?)?; diff --git a/h3ronpy/src/resolution.rs b/h3ronpy/src/resolution.rs index 9014f4f..7be75be 100644 --- a/h3ronpy/src/resolution.rs +++ b/h3ronpy/src/resolution.rs @@ -4,8 +4,9 @@ use pyo3::prelude::*; pub struct PyResolution(Resolution); -impl<'py> FromPyObject<'py> for PyResolution { - fn extract_bound(ob: &Bound<'py, PyAny>) -> PyResult { +impl<'a, 'py> FromPyObject<'a, 'py> for PyResolution { + type Error = PyErr; + fn extract(ob: Borrowed<'a, 'py, PyAny>) -> Result { let int = ob.extract::()?; let res = Resolution::try_from(int).map_err(|err| PyValueError::new_err(err.to_string()))?; diff --git a/h3ronpy/src/transform.rs b/h3ronpy/src/transform.rs index 482a8f0..75bc43e 100644 --- a/h3ronpy/src/transform.rs +++ b/h3ronpy/src/transform.rs @@ -5,7 +5,7 @@ use pyo3::prelude::*; use rasterh3::transform::{from_gdal, from_rasterio}; /// affine geotransform -#[pyclass] +#[pyclass(from_py_object)] #[derive(Clone)] pub struct Transform { pub(crate) inner: AffineTransform, diff --git a/h3ronpy/src/vector.rs b/h3ronpy/src/vector.rs index 9696752..66ad24c 100644 --- a/h3ronpy/src/vector.rs +++ b/h3ronpy/src/vector.rs @@ -1,5 +1,8 @@ use std::sync::Arc; +use crate::array::{PyCellArray, PyDirectedEdgeArray, PyVertexArray}; +use crate::arrow_interop::*; +use crate::error::IntoPyResult; use arrow::array::{ ArrayRef, AsArray, Float64Array, GenericBinaryArray, GenericListArray, OffsetSizeTrait, RecordBatch, UInt8Array, @@ -7,15 +10,16 @@ use arrow::array::{ use arrow::buffer::NullBuffer; use arrow::datatypes::{DataType, Field, Schema}; use geo::{BoundingRect, HasDimensions, LineString, ToRadians}; +use geoarrow_array::capacity::WkbCapacity; +use geoarrow_array::GeoArrowArray; use h3arrow::algorithm::ToCoordinatesOp; use h3arrow::array::from_geo::{ToCellIndexArray, ToCellListArray, ToCellsOptions}; use h3arrow::array::to_geoarrow::{ToWKBLineStrings, ToWKBPoints, ToWKBPolygons}; use h3arrow::array::{CellIndexArray, ResolutionArray}; -use h3arrow::export::geoarrow::array::{WKBArray, WKBBuilder, WKBCapacity}; -use h3arrow::export::geoarrow::ArrayBase; +use h3arrow::export::geoarrow::array::{GenericWkbArray, LargeWkbArray, WkbBuilder}; use h3arrow::export::h3o::geom::ContainmentMode; use h3arrow::export::h3o::Resolution; -use h3arrow::h3o::geom::dissolve; +use h3arrow::h3o::geom::SolventBuilder; use h3arrow::h3o::LatLng; use itertools::multizip; use pyo3::exceptions::PyValueError; @@ -24,10 +28,6 @@ use pyo3::types::PyTuple; use pyo3_arrow::error::PyArrowResult; use pyo3_arrow::{PyArray, PyRecordBatch}; -use crate::array::{PyCellArray, PyDirectedEdgeArray, PyVertexArray}; -use crate::arrow_interop::*; -use crate::error::IntoPyResult; - /// Containment mode used to decide if a cell is contained in a polygon or not. /// /// Modes: @@ -59,21 +59,16 @@ use crate::error::IntoPyResult; /// * Covers: This mode behaves the same as IntersectsBoundary, but also handles the case where the geometry is /// being covered by a cell without intersecting with its boundaries. In such cases, the covering cell is returned. /// -#[pyclass(name = "ContainmentMode", eq, eq_int)] -#[derive(Copy, Clone, Eq, PartialEq)] +#[pyclass(name = "ContainmentMode", eq, eq_int, hash, frozen, from_py_object)] +#[derive(Copy, Clone, Eq, PartialEq, Hash, Default)] pub enum PyContainmentMode { + #[default] ContainsCentroid, ContainsBoundary, IntersectsBoundary, Covers, } -impl Default for PyContainmentMode { - fn default() -> Self { - Self::ContainsCentroid - } -} - impl PyContainmentMode { fn containment_mode(&self) -> ContainmentMode { match self { @@ -87,11 +82,13 @@ impl PyContainmentMode { #[pyfunction] #[pyo3(signature = (cellarray,))] -pub(crate) fn cells_bounds(py: Python<'_>, cellarray: PyCellArray) -> PyResult> { - if let Some(rect) = py.allow_threads(|| cellarray.as_ref().bounding_rect()) { +pub(crate) fn cells_bounds( + py: Python<'_>, + cellarray: PyCellArray, +) -> PyResult>> { + if let Some(rect) = py.detach(|| cellarray.as_ref().bounding_rect()) { Ok(Some( - PyTuple::new_bound(py, [rect.min().x, rect.min().y, rect.max().x, rect.max().y]) - .to_object(py), + PyTuple::new(py, [rect.min().x, rect.min().y, rect.max().x, rect.max().y])?.into_any(), )) } else { Ok(None) @@ -100,7 +97,10 @@ pub(crate) fn cells_bounds(py: Python<'_>, cellarray: PyCellArray) -> PyResult PyArrowResult { +pub(crate) fn cells_bounds_arrays( + py: Python<'_>, + cellarray: PyCellArray, +) -> PyArrowResult> { let cellindexarray = cellarray.into_inner(); let mut minx_vec = vec![0.0f64; cellindexarray.len()]; let mut miny_vec = vec![0.0f64; cellindexarray.len()]; @@ -142,16 +142,16 @@ pub(crate) fn cells_bounds_arrays(py: Python, cellarray: PyCellArray) -> PyArrow Arc::new(Float64Array::new(maxy_vec.into(), Some(validity.clone()))), ]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } #[pyfunction] #[pyo3(signature = (cellarray, radians = false))] pub(crate) fn cells_to_coordinates( - py: Python, + py: Python<'_>, cellarray: PyCellArray, radians: bool, -) -> PyArrowResult { +) -> PyArrowResult> { let coordinate_arrays = if radians { cellarray.as_ref().to_coordinates_radians() } else { @@ -168,18 +168,18 @@ pub(crate) fn cells_to_coordinates( Arc::new(coordinate_arrays.lng), ]; let batch = RecordBatch::try_new(Arc::new(schema), columns)?; - Ok(PyRecordBatch::new(batch).to_arro3(py)?) + Ok(PyRecordBatch::new(batch).into_arro3(py)?) } #[pyfunction] #[pyo3(signature = (latarray, lngarray, resolution, radians = false))] -pub(crate) fn coordinates_to_cells( - py: Python<'_>, - latarray: &Bound, - lngarray: &Bound, - resolution: &Bound, +pub(crate) fn coordinates_to_cells<'py>( + py: Python<'py>, + latarray: &Bound<'py, PyAny>, + lngarray: &Bound<'py, PyAny>, + resolution: &Bound<'py, PyAny>, radians: bool, -) -> PyResult { +) -> PyResult> { let latarray: Float64Array = pyarray_to_native(latarray)?; let lngarray: Float64Array = pyarray_to_native(lngarray)?; if lngarray.len() != latarray.len() { @@ -191,7 +191,7 @@ pub(crate) fn coordinates_to_cells( let cells = if let Ok(resolution) = resolution.extract::() { let resolution = Resolution::try_from(resolution).into_pyresult()?; - py.allow_threads(|| { + py.detach(|| { latarray .iter() .zip(lngarray.iter()) @@ -219,7 +219,7 @@ pub(crate) fn coordinates_to_cells( )); } - py.allow_threads(|| { + py.detach(|| { multizip((latarray.iter(), lngarray.iter(), resarray.iter())) .map(|(lat, lng, res)| { if let (Some(lat), Some(lng), Some(res)) = (lat, lng, res) { @@ -243,21 +243,26 @@ pub(crate) fn coordinates_to_cells( #[pyfunction] #[pyo3(signature = (cellarray, radians = false, link_cells = false))] pub(crate) fn cells_to_wkb_polygons( - py: Python, + py: Python<'_>, cellarray: PyCellArray, radians: bool, link_cells: bool, -) -> PyResult { +) -> PyResult> { let cellindexarray = cellarray.into_inner(); let use_degrees = !radians; - let out: WKBArray = py.allow_threads(|| { + // LargeWkbArray (i64 offsets) to match the other WKB-producing functions, + // and to avoid i32 offset overflow on large outputs. + let out: LargeWkbArray = py.detach(|| { if link_cells { let mut cells = cellindexarray.iter().flatten().collect::>(); cells.sort_unstable(); cells.dedup(); - let geoms = dissolve(cells) + let solvent = SolventBuilder::default().build(); + + let geoms = solvent + .dissolve(cells) .into_pyresult()? .into_iter() .map(|mut poly| { @@ -267,74 +272,66 @@ pub(crate) fn cells_to_wkb_polygons( Some(geo_types::Geometry::from(poly)) }) .collect::>(); - let mut builder = WKBBuilder::with_capacity(WKBCapacity::from_geometries( - geoms.iter().map(|v| v.as_ref()), - )); - builder.extend_from_iter(geoms.iter().map(|v| v.as_ref())); + let mut builder = WkbBuilder::with_capacity( + Default::default(), + WkbCapacity::from_geometries(geoms.iter().map(|v| v.as_ref())), + ); + builder + .extend_from_iter(geoms.iter().map(|v| v.as_ref())) + .into_pyresult()?; Ok::<_, PyErr>(builder.finish()) } else { - Ok(cellindexarray - .to_wkb_polygons(use_degrees) - .expect("wkbarray")) + cellindexarray.to_wkb_polygons(use_degrees).into_pyresult() } })?; - let field = out.extension_field(); - PyArray::new(out.into_array_ref(), field).to_arro3(py) + let field = Arc::new(out.data_type().to_field("geometry", true)); + PyArray::new(out.into_array_ref(), field).into_arro3(py) } #[pyfunction] #[pyo3(signature = (cellarray, radians = false))] pub(crate) fn cells_to_wkb_points( - py: Python, + py: Python<'_>, cellarray: PyCellArray, radians: bool, -) -> PyResult { - let out = py.allow_threads(|| { - cellarray - .as_ref() - .to_wkb_points::(!radians) - .expect("wkbarray") - }); - - let field = out.extension_field(); - PyArray::new(out.into_array_ref(), field).to_arro3(py) +) -> PyResult> { + let out = py + .detach(|| cellarray.as_ref().to_wkb_points::(!radians)) + .into_pyresult()?; + + let field = Arc::new(out.data_type().to_field("geometry", true)); + PyArray::new(out.into_array_ref(), field).into_arro3(py) } #[pyfunction] #[pyo3(signature = (vertexarray, radians = false))] pub(crate) fn vertexes_to_wkb_points( - py: Python, + py: Python<'_>, vertexarray: PyVertexArray, radians: bool, -) -> PyResult { - let out = py.allow_threads(|| { - vertexarray - .as_ref() - .to_wkb_points::(!radians) - .expect("wkbarray") - }); - - let field = out.extension_field(); - PyArray::new(out.into_array_ref(), field).to_arro3(py) +) -> PyResult> { + let out = py + .detach(|| vertexarray.as_ref().to_wkb_points::(!radians)) + .into_pyresult()?; + + let field = Arc::new(out.data_type().to_field("geometry", true)); + PyArray::new(out.into_array_ref(), field).into_arro3(py) } #[pyfunction] #[pyo3(signature = (array, radians = false))] pub(crate) fn directededges_to_wkb_linestrings( - py: Python, + py: Python<'_>, array: PyDirectedEdgeArray, radians: bool, -) -> PyResult { - let out = py.allow_threads(|| { - array - .as_ref() - .to_wkb_linestrings::(!radians) - .expect("wkbarray") - }); - - let field = out.extension_field(); - PyArray::new(out.into_array_ref(), field).to_arro3(py) +) -> PyResult> { + let out = py + .detach(|| array.as_ref().to_wkb_linestrings::(!radians)) + .into_pyresult()?; + + let field = Arc::new(out.data_type().to_field("geometry", true)); + PyArray::new(out.into_array_ref(), field).into_arro3(py) } fn get_to_cells_options( @@ -352,13 +349,13 @@ fn get_to_cells_options( #[pyfunction] #[pyo3(signature = (array, resolution, containment_mode = None, compact = false, flatten = false))] pub(crate) fn wkb_to_cells( - py: Python, + py: Python<'_>, array: PyArray, resolution: u8, containment_mode: Option, compact: bool, flatten: bool, -) -> PyResult { +) -> PyResult> { let options = get_to_cells_options(resolution, containment_mode, compact)?; match array.field().data_type() { @@ -380,26 +377,26 @@ pub(crate) fn wkb_to_cells( } } -fn generic_wkb_to_cells( - py: Python, +fn generic_wkb_to_cells<'py, O: OffsetSizeTrait>( + py: Python<'py>, binarray: GenericBinaryArray, flatten: bool, options: &ToCellsOptions, -) -> PyResult { - let wkbarray = WKBArray::new(binarray, Default::default()); +) -> PyResult> { + let wkbarray = GenericWkbArray::::new(binarray, Default::default()); if flatten { let cells = py - .allow_threads(|| wkbarray.to_cellindexarray(options)) + .detach(|| wkbarray.to_cellindexarray(options)) .into_pyresult()?; h3array_to_pyarray(cells, py) } else { let listarray: GenericListArray = py - .allow_threads(|| wkbarray.to_celllistarray(options)) + .detach(|| wkbarray.to_celllistarray(options)) .into_pyresult()? .into(); - PyArray::from_array_ref(Arc::new(listarray)).to_arro3(py) + PyArray::from_array_ref(Arc::new(listarray)).into_arro3(py) } } @@ -411,12 +408,12 @@ pub(crate) fn geometry_to_cells( resolution: u8, containment_mode: Option, compact: bool, -) -> PyResult { +) -> PyResult> { if obj.0.is_empty() { return h3array_to_pyarray(CellIndexArray::new_null(0), py); } let options = get_to_cells_options(resolution, containment_mode, compact)?; - let cellindexarray = py.allow_threads(|| { + let cellindexarray = py.detach(|| { Ok::<_, PyErr>(CellIndexArray::from( h3arrow::array::from_geo::geometry_to_cells(&obj.0, &options).into_pyresult()?, )) diff --git a/h3ronpy/tests/arrow/test_vector.py b/h3ronpy/tests/arrow/test_vector.py index 5152301..03efc4d 100644 --- a/h3ronpy/tests/arrow/test_vector.py +++ b/h3ronpy/tests/arrow/test_vector.py @@ -1,7 +1,18 @@ import h3.api.numpy_int as h3 +import numpy as np +import pytest import shapely from arro3.core import Array, DataType, Scalar -from h3ronpy.vector import ContainmentMode, cells_to_wkb_points, geometry_to_cells +from h3ronpy import uncompact +from h3ronpy.vector import ( + ContainmentMode, + cells_to_wkb_points, + cells_to_wkb_polygons, + directededges_to_wkb_linestrings, + geometry_to_cells, + vertexes_to_wkb_points, + wkb_to_cells, +) from shapely import wkb from shapely.geometry import Point @@ -43,3 +54,94 @@ def test_coordinate_values_are_not_equal_issue_58(): shapely_point = wkb.loads(wkb_point.as_py()) assert int(lat) == int(shapely_point.y) assert int(lon) == int(shapely_point.x) + + +SOME_POLYGON = shapely.Polygon(((10.0, 45.0), (10.0, 46.0), (11.0, 46.0), (11.0, 45.0), (10.0, 45.0))) + + +def test_wkb_outputs_are_large_binary(): + # regression test: with geoarrow-array 0.8, the `WkbArray` alias uses i32 + # offsets. All WKB-producing functions must keep returning large_binary + # (i64 offsets). + cells = geometry_to_cells(SOME_POLYGON, 5) + assert len(cells) > 0 + + assert cells_to_wkb_polygons(cells).type == DataType.large_binary() + assert cells_to_wkb_polygons(cells, link_cells=True).type == DataType.large_binary() + assert cells_to_wkb_points(cells).type == DataType.large_binary() + + some_cell = h3.latlng_to_cell(45.5, 10.2, 5) + vertexes = np.array([h3.cell_to_vertex(some_cell, v) for v in range(6)], dtype=np.uint64) + assert vertexes_to_wkb_points(vertexes).type == DataType.large_binary() + + edges = np.array(h3.origin_to_directed_edges(some_cell), dtype=np.uint64) + assert directededges_to_wkb_linestrings(edges).type == DataType.large_binary() + + +def test_wkb_polygons_field_name_and_extension_metadata(): + cells = geometry_to_cells(SOME_POLYGON, 5) + field = cells_to_wkb_polygons(cells).field + + assert field.name == "geometry" + metadata = { + (k.decode() if isinstance(k, bytes) else k): (v.decode() if isinstance(v, bytes) else v) + for k, v in field.metadata.items() + } + assert metadata.get("ARROW:extension:name") == "geoarrow.wkb" + + +def test_wkb_to_cells_flatten_and_compact(): + wkb_array = Array([shapely.to_wkb(SOME_POLYGON)], type=DataType.binary()) + + list_result = wkb_to_cells(wkb_array, 7) + flat_result = wkb_to_cells(wkb_array, 7, flatten=True) + + assert flat_result.type == DataType.uint64() + cells_from_list = [cell for cells in list_result.to_pylist() for cell in cells] + assert sorted(cells_from_list) == sorted(flat_result.to_pylist()) + + compact_lists = wkb_to_cells(wkb_array, 7, compact=True) + compact_cells = [cell for cells in compact_lists.to_pylist() for cell in cells] + assert 0 < len(compact_cells) < len(flat_result) + # uncompacting must restore the exact same cell set + compact_array = Array(compact_cells, type=DataType.uint64()) + assert sorted(uncompact(compact_array, 7).to_pylist()) == sorted(flat_result.to_pylist()) + + +@pytest.mark.xfail( + reason=( + "compact+flatten compacts the already per-geometry-compacted (mixed resolution) " + "cells a second time and fails with 'heterogeneous resolution'. Present in 0.22.0 as well." + ), + strict=True, +) +def test_wkb_to_cells_compact_and_flatten_combined(): + wkb_array = Array([shapely.to_wkb(SOME_POLYGON)], type=DataType.binary()) + wkb_to_cells(wkb_array, 7, compact=True, flatten=True) + + +def test_wkb_to_cells_containment_modes(): + wkb_array = Array([shapely.to_wkb(SOME_POLYGON)], type=DataType.binary()) + + counts = { + mode: len(wkb_to_cells(wkb_array, 6, containment_mode=mode, flatten=True)) + for mode in ( + ContainmentMode.ContainsBoundary, + ContainmentMode.ContainsCentroid, + ContainmentMode.IntersectsBoundary, + ContainmentMode.Covers, + ) + } + assert all(count > 0 for count in counts.values()) + assert counts[ContainmentMode.ContainsBoundary] <= counts[ContainmentMode.IntersectsBoundary] + assert counts[ContainmentMode.ContainsCentroid] <= counts[ContainmentMode.IntersectsBoundary] + assert counts[ContainmentMode.IntersectsBoundary] <= counts[ContainmentMode.Covers] + + +def test_wkb_to_cells_null_entries_are_preserved(): + wkb_array = Array([shapely.to_wkb(SOME_POLYGON), None], type=DataType.binary()) + + result = wkb_to_cells(wkb_array, 5).to_pylist() + assert len(result) == 2 + assert len(result[0]) > 0 + assert result[1] is None