From 992b76764cfe92e9c321443a1f5f595936d02b2f Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:38:16 +0000 Subject: [PATCH 1/2] feat(csharp): define geospatial Arrow types Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- csharp/src/GeospatialArrowType.cs | 260 +++++++++++++++++++ csharp/test/Unit/GeospatialArrowTypeTests.cs | 139 ++++++++++ 2 files changed, 399 insertions(+) create mode 100644 csharp/src/GeospatialArrowType.cs create mode 100644 csharp/test/Unit/GeospatialArrowTypeTests.cs diff --git a/csharp/src/GeospatialArrowType.cs b/csharp/src/GeospatialArrowType.cs new file mode 100644 index 00000000..f8cffa6c --- /dev/null +++ b/csharp/src/GeospatialArrowType.cs @@ -0,0 +1,260 @@ +/* +* Copyright (c) 2026 ADBC Drivers Contributors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.RegularExpressions; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Types; + +namespace AdbcDrivers.Databricks +{ + /// + /// Describes Databricks GEOMETRY / GEOGRAPHY values, recognizes Reyden's + /// transport representation, and builds the public GeoArrow field metadata. + /// + internal static class GeospatialArrowType + { + internal const string GeometryMetadataKey = "geometry"; + internal const string GeographyMetadataKey = "geography"; + internal const string SridMetadataKey = "srid"; + internal const string ExtensionNameMetadataKey = "ARROW:extension:name"; + internal const string ExtensionMetadataKey = "ARROW:extension:metadata"; + internal const string WkbExtensionName = "geoarrow.wkb"; + + private static readonly Regex s_typePattern = new Regex( + @"^\s*(GEOMETRY|GEOGRAPHY)\s*(?:\(\s*(ANY|\d+)\s*\))?\s*$", + RegexOptions.IgnoreCase | RegexOptions.CultureInvariant); + + internal enum Family + { + Geometry, + Geography, + } + + internal readonly struct Tag : IEquatable + { + internal Tag(Family family, int srid) + { + Family = family; + Srid = srid; + } + + internal Family Family { get; } + + internal int Srid { get; } + + public bool Equals(Tag other) => Family == other.Family && Srid == other.Srid; + + public override bool Equals(object? obj) => obj is Tag other && Equals(other); + + public override int GetHashCode() => ((int)Family * 397) ^ Srid; + } + + internal static bool TryCreate(string typeText, out StructType type) + { + type = null!; + Match match = s_typePattern.Match(typeText ?? string.Empty); + if (!match.Success) + return false; + + Family family = match.Groups[1].Value.Equals( + "GEOGRAPHY", + StringComparison.OrdinalIgnoreCase) + ? Family.Geography + : Family.Geometry; + if (!TryParseTypeSrid(match, family, out int srid)) + return false; + + type = Create(new Tag(family, srid)); + return true; + } + + internal static StructType Create(Tag tag) + { + var metadata = new Dictionary + { + [tag.Family == Family.Geometry ? GeometryMetadataKey : GeographyMetadataKey] = "true", + [SridMetadataKey] = tag.Srid.ToString(CultureInfo.InvariantCulture), + }; + return new StructType(new[] + { + new Field("srid", Int32Type.Default, nullable: false), + new Field("wkb", BinaryType.Default, nullable: false, metadata), + }); + } + + /// + /// Creates the public GeoArrow WKB field for a Databricks geospatial type. + /// Fixed positive SRIDs are advertised as opaque database SRIDs. Databricks + /// supports CRS authorities other than EPSG, and the manifest exposes only the + /// integer SRID. ANY has no column CRS because its SRID varies by row; those + /// rows carry EWKB instead. + /// + internal static Field CreateGeoArrowField(Field source, Tag tag) + { + var metadata = new Dictionary(); + if (source.Metadata != null) + { + foreach (KeyValuePair item in source.Metadata) + { + metadata[item.Key] = item.Value; + } + } + + metadata[ExtensionNameMetadataKey] = WkbExtensionName; + string? extensionMetadata = CreateGeoArrowMetadata(tag); + if (extensionMetadata == null) + { + metadata.Remove(ExtensionMetadataKey); + } + else + { + metadata[ExtensionMetadataKey] = extensionMetadata; + } + + return new Field( + source.Name, + BinaryType.Default, + source.IsNullable, + metadata); + } + + internal static bool IsStructShape(IArrowType type) + { + if (!(type is StructType structType) || structType.Fields.Count != 2) + return false; + + return structType.Fields[0].Name == "srid" + && structType.Fields[0].DataType.TypeId == ArrowTypeId.Int32 + && structType.Fields[1].Name == "wkb" + && structType.Fields[1].DataType.TypeId == ArrowTypeId.Binary; + } + + /// + /// Resolves the canonical geo tag when present. An exact untagged + /// struct<srid,wkb> returns false. Partial or conflicting tags throw because + /// they represent malformed server metadata. + /// + internal static bool ResolveTag(IArrowType type, out Tag tag) + { + tag = default; + if (!IsStructShape(type)) + return false; + + StructType structType = (StructType)type; + IReadOnlyDictionary? metadata = structType.Fields[1].Metadata; + bool hasGeometry = metadata?.ContainsKey(GeometryMetadataKey) == true; + bool hasGeography = metadata?.ContainsKey(GeographyMetadataKey) == true; + bool geometry = MetadataIsTrue(metadata, GeometryMetadataKey); + bool geography = MetadataIsTrue(metadata, GeographyMetadataKey); + bool hasMetadata = hasGeometry + || hasGeography + || metadata?.ContainsKey(SridMetadataKey) == true; + if (!hasMetadata) + return false; + + if ((hasGeometry && !geometry) || (hasGeography && !geography)) + { + throw InvalidTag("geometry and geography metadata values must be true"); + } + + if (geometry == geography) + { + throw InvalidTag( + "expected exactly one of geometry=true or geography=true"); + } + + if (metadata == null + || !metadata.TryGetValue(SridMetadataKey, out string? sridText) + || !int.TryParse( + sridText, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int srid)) + { + throw InvalidTag("missing or invalid type-level srid"); + } + + if (srid < -1) + { + throw InvalidTag("type-level srid must be -1 (ANY) or nonnegative"); + } + + tag = new Tag(geometry ? Family.Geometry : Family.Geography, srid); + return true; + } + + private static bool TryParseTypeSrid( + Match match, + Family family, + out int srid) + { + if (!match.Groups[2].Success) + { + // Bare SQL types use their Databricks defaults. ANY is distinct + // and uses Reyden's -1 type-level sentinel. + srid = family == Family.Geography ? 4326 : 0; + return true; + } + + if (match.Groups[2].Value.Equals("ANY", StringComparison.OrdinalIgnoreCase)) + { + srid = -1; + return true; + } + + return int.TryParse( + match.Groups[2].Value, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out srid) + && srid >= 0; + } + + private static bool MetadataIsTrue( + IReadOnlyDictionary? metadata, + string key) => + metadata != null + && metadata.TryGetValue(key, out string? value) + && value.Equals("true", StringComparison.OrdinalIgnoreCase); + + private static string? CreateGeoArrowMetadata(Tag tag) + { + string? crs = tag.Srid > 0 + ? $"\"crs\":\"{tag.Srid.ToString(CultureInfo.InvariantCulture)}\",\"crs_type\":\"srid\"" + : null; + string? edges = tag.Family == Family.Geography + ? "\"edges\":\"spherical\"" + : null; + + if (crs == null && edges == null) + return null; + if (crs == null) + return $"{{{edges}}}"; + if (edges == null) + return $"{{{crs}}}"; + return $"{{{crs},{edges}}}"; + } + + private static DatabricksException InvalidTag(string detail) => + new DatabricksException( + $"Malformed geospatial Arrow tag: {detail}", + AdbcStatusCode.InvalidData); + } +} diff --git a/csharp/test/Unit/GeospatialArrowTypeTests.cs b/csharp/test/Unit/GeospatialArrowTypeTests.cs new file mode 100644 index 00000000..9c6629ce --- /dev/null +++ b/csharp/test/Unit/GeospatialArrowTypeTests.cs @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Types; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit +{ + public class GeospatialArrowTypeTests + { + [Theory] + [InlineData("GEOMETRY", "geometry", "0")] + [InlineData("GEOMETRY(3857)", "geometry", "3857")] + [InlineData("GEOMETRY(ANY)", "geometry", "-1")] + [InlineData("GEOGRAPHY", "geography", "4326")] + [InlineData("GEOGRAPHY(4326)", "geography", "4326")] + public void TryCreate_ReturnsTaggedInternalStruct( + string sqlType, + string familyKey, + string expectedSrid) + { + Assert.True(GeospatialArrowType.TryCreate(sqlType, out StructType type)); + + Assert.Equal(2, type.Fields.Count); + Assert.Equal("srid", type.Fields[0].Name); + Assert.IsType(type.Fields[0].DataType); + Assert.False(type.Fields[0].IsNullable); + Assert.Equal("wkb", type.Fields[1].Name); + Assert.IsType(type.Fields[1].DataType); + Assert.False(type.Fields[1].IsNullable); + Assert.Equal("true", type.Fields[1].Metadata[familyKey]); + Assert.Equal(expectedSrid, type.Fields[1].Metadata["srid"]); + } + + [Theory] + [InlineData("GEOMETRY(foo)")] + [InlineData("GEOGRAPHY()")] + [InlineData("GEOMETRY(-1)")] + [InlineData("GEOGRAPHY(-2)")] + [InlineData("GEOMETRY(2147483648)")] + [InlineData("GEOGRAPHY(4326) trailing")] + public void TryCreate_RejectsMalformedType(string sqlType) + { + Assert.False(GeospatialArrowType.TryCreate(sqlType, out _)); + } + + [Fact] + public void PartialArrowTag_IsRejected() + { + StructType type = TaggedType( + new Dictionary { ["geometry"] = "true" }); + + DatabricksException error = Assert.Throws(() => + GeospatialArrowType.ResolveTag(type, out _)); + Assert.Equal(AdbcStatusCode.InvalidData, error.Status); + Assert.Contains("srid", error.Message, System.StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ConflictingArrowTag_IsRejected() + { + StructType type = TaggedType(new Dictionary + { + ["geometry"] = "true", + ["geography"] = "true", + ["srid"] = "4326", + }); + + DatabricksException error = Assert.Throws(() => + GeospatialArrowType.ResolveTag(type, out _)); + Assert.Equal(AdbcStatusCode.InvalidData, error.Status); + Assert.Contains("exactly one", error.Message, System.StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void NegativeFixedSridArrowTag_IsRejected() + { + StructType type = TaggedType(new Dictionary + { + ["geometry"] = "true", + ["srid"] = "-2", + }); + + DatabricksException error = Assert.Throws(() => + GeospatialArrowType.ResolveTag(type, out _)); + Assert.Equal(AdbcStatusCode.InvalidData, error.Status); + Assert.Contains("nonnegative", error.Message, System.StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void UntaggedTransportShape_IsNotGeospatial() + { + StructType type = TaggedType(new Dictionary()); + + Assert.False(GeospatialArrowType.ResolveTag(type, out _)); + } + + [Theory] + [InlineData("GEOMETRY(103192)", "{\"crs\":\"103192\",\"crs_type\":\"srid\"}")] + [InlineData("GEOGRAPHY(4326)", "{\"crs\":\"4326\",\"crs_type\":\"srid\",\"edges\":\"spherical\"}")] + public void GeoArrowMetadata_TreatsSridAsOpaque( + string sqlType, + string expectedMetadata) + { + Assert.True(GeospatialArrowType.TryCreate(sqlType, out StructType type)); + Assert.True(GeospatialArrowType.ResolveTag(type, out GeospatialArrowType.Tag tag)); + + Field output = GeospatialArrowType.CreateGeoArrowField( + new Field("geo", type, nullable: true), + tag); + + Assert.Equal("geoarrow.wkb", output.Metadata["ARROW:extension:name"]); + Assert.Equal(expectedMetadata, output.Metadata["ARROW:extension:metadata"]); + } + + private static StructType TaggedType(Dictionary metadata) => + new StructType(new[] + { + new Field("srid", Int32Type.Default, nullable: false), + new Field("wkb", BinaryType.Default, nullable: false, metadata), + }); + } +} From 2cbc38a6a20b8169749aa97f01b3b1afa8d48dc7 Mon Sep 17 00:00:00 2001 From: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> Date: Fri, 16 Oct 2026 19:02:08 +0000 Subject: [PATCH 2/2] fix(csharp): validate geospatial Arrow type metadata Signed-off-by: Cathleen Yan <58714163+cathleeny@users.noreply.github.com> --- csharp/src/GeospatialArrowType.cs | 11 +- csharp/test/Unit/GeospatialArrowTypeTests.cs | 101 ++++++++++++++++++- 2 files changed, 106 insertions(+), 6 deletions(-) diff --git a/csharp/src/GeospatialArrowType.cs b/csharp/src/GeospatialArrowType.cs index f8cffa6c..f4a76e0c 100644 --- a/csharp/src/GeospatialArrowType.cs +++ b/csharp/src/GeospatialArrowType.cs @@ -104,7 +104,8 @@ internal static StructType Create(Tag tag) /// Fixed positive SRIDs are advertised as opaque database SRIDs. Databricks /// supports CRS authorities other than EPSG, and the manifest exposes only the /// integer SRID. ANY has no column CRS because its SRID varies by row; those - /// rows carry EWKB instead. + /// rows carry EWKB instead. GeoArrow readers may reject EWKB or ignore its + /// embedded SRID; preserving those bytes does not guarantee reader support. /// internal static Field CreateGeoArrowField(Field source, Tag tag) { @@ -196,6 +197,11 @@ internal static bool ResolveTag(IArrowType type, out Tag tag) throw InvalidTag("type-level srid must be -1 (ANY) or nonnegative"); } + if (geography && srid != -1 && srid != 4326) + { + throw InvalidTag("GEOGRAPHY type-level srid must be -1 (ANY) or 4326"); + } + tag = new Tag(geometry ? Family.Geometry : Family.Geography, srid); return true; } @@ -224,7 +230,8 @@ private static bool TryParseTypeSrid( NumberStyles.Integer, CultureInfo.InvariantCulture, out srid) - && srid >= 0; + && srid >= 0 + && (family != Family.Geography || srid == 4326); } private static bool MetadataIsTrue( diff --git a/csharp/test/Unit/GeospatialArrowTypeTests.cs b/csharp/test/Unit/GeospatialArrowTypeTests.cs index 9c6629ce..18f3c9a1 100644 --- a/csharp/test/Unit/GeospatialArrowTypeTests.cs +++ b/csharp/test/Unit/GeospatialArrowTypeTests.cs @@ -26,10 +26,12 @@ public class GeospatialArrowTypeTests { [Theory] [InlineData("GEOMETRY", "geometry", "0")] + [InlineData("GEOMETRY(0)", "geometry", "0")] [InlineData("GEOMETRY(3857)", "geometry", "3857")] [InlineData("GEOMETRY(ANY)", "geometry", "-1")] [InlineData("GEOGRAPHY", "geography", "4326")] [InlineData("GEOGRAPHY(4326)", "geography", "4326")] + [InlineData("GEOGRAPHY(ANY)", "geography", "-1")] public void TryCreate_ReturnsTaggedInternalStruct( string sqlType, string familyKey, @@ -46,6 +48,7 @@ public void TryCreate_ReturnsTaggedInternalStruct( Assert.False(type.Fields[1].IsNullable); Assert.Equal("true", type.Fields[1].Metadata[familyKey]); Assert.Equal(expectedSrid, type.Fields[1].Metadata["srid"]); + Assert.True(GeospatialArrowType.ResolveTag(type, out _)); } [Theory] @@ -53,6 +56,9 @@ public void TryCreate_ReturnsTaggedInternalStruct( [InlineData("GEOGRAPHY()")] [InlineData("GEOMETRY(-1)")] [InlineData("GEOGRAPHY(-2)")] + [InlineData("GEOGRAPHY(0)")] + [InlineData("GEOGRAPHY(3857)")] + [InlineData("GEOGRAPHY(2147483647)")] [InlineData("GEOMETRY(2147483648)")] [InlineData("GEOGRAPHY(4326) trailing")] public void TryCreate_RejectsMalformedType(string sqlType) @@ -111,22 +117,109 @@ public void UntaggedTransportShape_IsNotGeospatial() Assert.False(GeospatialArrowType.ResolveTag(type, out _)); } + [Theory] + [InlineData("0")] + [InlineData("3857")] + [InlineData("2147483647")] + public void UnsupportedFixedGeographySridArrowTag_IsRejected(string srid) + { + StructType type = TaggedType(new Dictionary + { + ["geography"] = "true", + ["srid"] = srid, + }); + + DatabricksException error = Assert.Throws(() => + GeospatialArrowType.ResolveTag(type, out _)); + Assert.Equal(AdbcStatusCode.InvalidData, error.Status); + Assert.Contains("4326", error.Message); + } + + [Theory] + [InlineData("geometry", "false", "4326", "must be true")] + [InlineData("geography", "false", "4326", "must be true")] + [InlineData("geometry", "true", "not-an-integer", "invalid type-level srid")] + [InlineData("geometry", "true", "2147483648", "invalid type-level srid")] + [InlineData(null, null, "4326", "exactly one")] + public void MalformedArrowTag_IsRejected( + string? familyKey, + string? familyValue, + string srid, + string expectedError) + { + var metadata = new Dictionary { ["srid"] = srid }; + if (familyKey != null) + metadata[familyKey] = familyValue!; + + DatabricksException error = Assert.Throws(() => + GeospatialArrowType.ResolveTag(TaggedType(metadata), out _)); + Assert.Equal(AdbcStatusCode.InvalidData, error.Status); + Assert.Contains(expectedError, error.Message); + } + + [Fact] + public void NearMissTransportShapes_AreNotGeospatial() + { + StructType canonical = TaggedType(new Dictionary + { + ["geometry"] = "true", + ["srid"] = "4326", + }); + Field srid = canonical.Fields[0]; + Field wkb = canonical.Fields[1]; + var nearMisses = new[] + { + new StructType(new[] + { + srid, + new Field("wkb", LargeBinaryType.Default, nullable: false, wkb.Metadata), + }), + new StructType(new[] { wkb, srid }), + new StructType(new[] { srid, wkb, new Field("extra", Int32Type.Default, false) }), + }; + + foreach (StructType type in nearMisses) + { + Assert.False(GeospatialArrowType.IsStructShape(type)); + Assert.False(GeospatialArrowType.ResolveTag(type, out _)); + } + } + [Theory] [InlineData("GEOMETRY(103192)", "{\"crs\":\"103192\",\"crs_type\":\"srid\"}")] [InlineData("GEOGRAPHY(4326)", "{\"crs\":\"4326\",\"crs_type\":\"srid\",\"edges\":\"spherical\"}")] - public void GeoArrowMetadata_TreatsSridAsOpaque( + [InlineData("GEOMETRY", null)] + [InlineData("GEOMETRY(0)", null)] + [InlineData("GEOMETRY(ANY)", null)] + [InlineData("GEOGRAPHY(ANY)", "{\"edges\":\"spherical\"}")] + public void GeoArrowMetadata_ReplacesStaleMetadataForSridAndFamily( string sqlType, - string expectedMetadata) + string? expectedMetadata) { Assert.True(GeospatialArrowType.TryCreate(sqlType, out StructType type)); Assert.True(GeospatialArrowType.ResolveTag(type, out GeospatialArrowType.Tag tag)); + var sourceMetadata = new Dictionary + { + ["ARROW:extension:name"] = "stale.extension", + ["ARROW:extension:metadata"] = "{\"crs\":\"stale\"}", + ["custom"] = "preserved", + }; Field output = GeospatialArrowType.CreateGeoArrowField( - new Field("geo", type, nullable: true), + new Field("geo", type, nullable: true, sourceMetadata), tag); + Assert.Equal("geo", output.Name); + Assert.True(output.IsNullable); + Assert.IsType(output.DataType); Assert.Equal("geoarrow.wkb", output.Metadata["ARROW:extension:name"]); - Assert.Equal(expectedMetadata, output.Metadata["ARROW:extension:metadata"]); + Assert.Equal("preserved", output.Metadata["custom"]); + if (expectedMetadata == null) + Assert.False(output.Metadata.ContainsKey("ARROW:extension:metadata")); + else + Assert.Equal(expectedMetadata, output.Metadata["ARROW:extension:metadata"]); + Assert.Equal("stale.extension", sourceMetadata["ARROW:extension:name"]); + Assert.Equal("{\"crs\":\"stale\"}", sourceMetadata["ARROW:extension:metadata"]); } private static StructType TaggedType(Dictionary metadata) =>