From 6b43bbd8ba2734a94b0f5f5a3c177f1d2ac4133b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 17:57:30 +0000 Subject: [PATCH 01/22] feat(csharp): support native SEA metadata results Signed-off-by: Vu Anh Phung --- .../NativeMetadataResultBuilder.cs | 177 ++++++++++++++++++ .../StatementExecutionClient.cs | 6 + .../StatementExecutionConnection.cs | 154 +++++++++------ .../StatementExecutionModels.cs | 16 ++ .../StatementExecutionStatement.cs | 103 +++++++--- .../NativeMetadataResultBuilderTests.cs | 67 +++++++ .../StatementExecutionClientTests.cs | 49 +++++ .../StatementExecutionEmptyTableTypesTests.cs | 95 ++++++++++ 8 files changed, 590 insertions(+), 77 deletions(-) create mode 100644 csharp/src/StatementExecution/NativeMetadataResultBuilder.cs create mode 100644 csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs new file mode 100644 index 00000000..01ab2a07 --- /dev/null +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Types; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal static class NativeMetadataResultBuilder + { + internal static QueryResult Build( + IReadOnlyList batches, Schema schema, MetadataOperation operation, + string? requestedCatalog = null, IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + { + // C# exposes BASE_TYPE_NAME after the 23 Thrift GetColumns fields. + int sourceColumns = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); + var rows = new List<(RecordBatch Batch, int Index)>(); + foreach (var batch in batches) + { + if (batch.Schema.FieldsList.Count != sourceColumns) + throw new DatabricksException($"Invalid native {operation} result: expected {sourceColumns} columns, found {batch.Schema.FieldsList.Count}"); + + for (int row = 0; row < batch.Length; row++) + { + if (operation == MetadataOperation.GetTables && requestedCatalog != null && + !string.Equals(requestedCatalog, ReadString(batch.Column(0), row), StringComparison.OrdinalIgnoreCase)) + continue; + if (operation == MetadataOperation.GetTables && tableTypes != null && + !tableTypes.Contains(ReadString(batch.Column(3), row) ?? string.Empty)) + continue; + + if (operation == MetadataOperation.GetCrossReference && + (!Matches(parentCatalog, ReadString(batch.Column(0), row)) || + !Matches(parentSchema, ReadString(batch.Column(1), row)) || + !Matches(parentTable, ReadString(batch.Column(2), row)))) + continue; + + rows.Add((batch, row)); + } + } + + if (operation == MetadataOperation.GetTables) + { + int[] sortColumns = { 3, 0, 1, 2 }; + rows.Sort((left, right) => + { + foreach (int column in sortColumns) + { + int result = StringComparer.Ordinal.Compare( + ReadString(left.Batch.Column(column), left.Index), + ReadString(right.Batch.Column(column), right.Index)); + if (result != 0) return result; + } + return 0; + }); + } + + var arrays = new List(schema.FieldsList.Count); + for (int column = 0; column < schema.FieldsList.Count; column++) + { + switch (schema.FieldsList[column].DataType.TypeId) + { + case ArrowTypeId.String: + var strings = new StringArray.Builder(); + foreach (var (batch, row) in rows) + { + string? typeName = column == sourceColumns && operation == MetadataOperation.GetColumns + ? ReadString(batch.Column(5), row) + : null; + string? value = column == sourceColumns && operation == MetadataOperation.GetColumns + ? typeName == null ? null : ColumnMetadataHelper.GetBaseTypeName(typeName) + : ReadString(batch.Column(column), row); + if (column == 1 && operation == MetadataOperation.GetSchemas) + value ??= requestedCatalog; + if (column == 0 && (operation == MetadataOperation.GetTables || operation == MetadataOperation.GetColumns)) + value ??= requestedCatalog; + if (value == null) strings.AppendNull(); else strings.Append(value); + } + arrays.Add(strings.Build()); + break; + + case ArrowTypeId.Int8: + var int8 = new Int8Array.Builder(); + foreach (var (batch, row) in rows) + { + long? value = ReadInteger(batch.Column(column), row); + if (value.HasValue) int8.Append(checked((sbyte)value.Value)); else int8.AppendNull(); + } + arrays.Add(int8.Build()); + break; + + case ArrowTypeId.Int16: + var int16 = new Int16Array.Builder(); + foreach (var (batch, row) in rows) + { + long? value = ReadInteger(batch.Column(column), row); + if (value.HasValue) int16.Append(checked((short)value.Value)); else int16.AppendNull(); + } + arrays.Add(int16.Build()); + break; + + case ArrowTypeId.Int32: + var int32 = new Int32Array.Builder(); + foreach (var (batch, row) in rows) + { + long? value = ReadInteger(batch.Column(column), row); + if (value.HasValue && operation == MetadataOperation.GetColumns && + schema.FieldsList[column].Name == "ORDINAL_POSITION") + value = checked(value.Value + 1); + if (value.HasValue) int32.Append(checked((int)value.Value)); else int32.AppendNull(); + } + arrays.Add(int32.Build()); + break; + + case ArrowTypeId.Int64: + var int64 = new Int64Array.Builder(); + foreach (var (batch, row) in rows) + { + long? value = ReadInteger(batch.Column(column), row); + if (value.HasValue) int64.Append(value.Value); else int64.AppendNull(); + } + arrays.Add(int64.Build()); + break; + + default: + throw new DatabricksException($"Unsupported native metadata field type: {schema.FieldsList[column].DataType}"); + } + } + + return new QueryResult(rows.Count, new HiveInfoArrowStream(schema, arrays.ToArray())); + } + + internal static string? ReadString(IArrowArray array, int row) + { + if (array.IsNull(row)) return null; + return array is StringArray strings + ? strings.GetString(row) + : throw new DatabricksException($"Expected a native metadata string, found {array.GetType().Name}"); + } + + internal static long? ReadInteger(IArrowArray array, int row) + { + if (array.IsNull(row)) return null; + return array switch + { + Int8Array values => values.GetValue(row), + Int16Array values => values.GetValue(row), + Int32Array values => values.GetValue(row), + Int64Array values => values.GetValue(row), + _ => throw new DatabricksException($"Expected a native metadata integer, found {array.GetType().Name}") + }; + } + + private static bool Matches(string? requested, string? actual) + => requested == null || (actual != null && string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/csharp/src/StatementExecution/StatementExecutionClient.cs b/csharp/src/StatementExecution/StatementExecutionClient.cs index b953fed7..9b1affd7 100644 --- a/csharp/src/StatementExecution/StatementExecutionClient.cs +++ b/csharp/src/StatementExecution/StatementExecutionClient.cs @@ -227,6 +227,12 @@ public async Task ExecuteStatementAsync( httpRequest.Headers.TryAddWithoutValidation("x-databricks-sea-can-run-fully-sync", "true"); } + if (request.IsMetadata && request.MetadataOperation is MetadataOperation operation) + { + httpRequest.Headers.TryAddWithoutValidation("x-databricks-metadata-operation-type", operation.ToString()); + httpRequest.Headers.TryAddWithoutValidation("x-databricks-require-thrift-native-metadata", "true"); + } + var response = await _httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false); await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index ce046f8c..24c566b1 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -710,13 +710,13 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string using var cts = CreateMetadataTimeoutCts(); // Pass catalog through with SPARK→null normalization, matching Thrift - // which sends catalog as-is to the server. ExecuteShowColumnsAsync + // which sends catalog as-is to the server. ExecuteNativeShowColumnsAsync // handles null by iterating all catalogs. string? resolvedCatalog = DatabricksConnection.HandleSparkCatalog(catalog); - List batches; + List<(RecordBatch Batch, bool IsNative)> batches; try { - batches = ExecuteShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) + batches = ExecuteNativeShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) .GetAwaiter().GetResult(); } catch (Exception ex) when (cts.IsCancellationRequested && ex is not TimeoutException) @@ -728,11 +728,11 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string } var fields = new List(); - foreach (var batch in batches) + foreach (var (batch, isNative) in batches) { - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); + var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); if (colNameArray == null || columnTypeArray == null) continue; @@ -743,7 +743,8 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string string colName = colNameArray.GetString(i); string colType = columnTypeArray.GetString(i); bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + !isNullableArray.GetString(i).Equals( + isNative ? "NO" : "false", StringComparison.OrdinalIgnoreCase); short typeCode = ColumnMetadataHelper.GetDataTypeCode(colType); IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, colType, false, null, null); @@ -762,9 +763,11 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin { string sql = new ShowCatalogsCommand(catalogPattern).Build(); List batches; + bool isNative; try { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -775,7 +778,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin var result = new List(); foreach (var batch in batches) { - var catalogArray = TryGetColumn(batch, "catalog"); + var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalog"); if (catalogArray == null) continue; for (int i = 0; i < batch.Length; i++) { @@ -795,9 +798,11 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); List batches; + bool isNative; try { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -814,7 +819,12 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin StringArray? catalogArray = null; StringArray? schemaArray = null; - if (showSchemasInAllCatalogs) + if (isNative) + { + schemaArray = batch.Column(0) as StringArray; + catalogArray = batch.Column(1) as StringArray; + } + else if (showSchemasInAllCatalogs) { schemaArray = batch.Column(0) as StringArray; catalogArray = batch.Column(1) as StringArray; @@ -853,27 +863,32 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); List batches; + bool isNative; try { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { return System.Array.Empty<(string, string, string, string)>(); } - var result = new List<(string, string, string, string)>(); + var result = new List<(string catalog, string schema, string table, string tableType)>(); foreach (var batch in batches) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var tableTypeArray = TryGetColumn(batch, "tableType"); + var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); + var tableArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); + var tableTypeArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "tableType"); if (catalogArray == null || schemaArray == null || tableArray == null) continue; for (int i = 0; i < batch.Length; i++) { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; + if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; + if (isNative && catalogPattern != null && + !string.Equals(catalogPattern, catalogArray.IsNull(i) ? null : catalogArray.GetString(i), + StringComparison.OrdinalIgnoreCase)) continue; string tableType = "TABLE"; if (tableTypeArray != null && !tableTypeArray.IsNull(i)) @@ -889,10 +904,22 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin if (tableTypes != null && !tableTypes.Contains(tableType)) continue; - result.Add((catalogArray.GetString(i), schemaArray.GetString(i), + result.Add((catalogArray.IsNull(i) ? catalogPattern ?? "" : catalogArray.GetString(i), schemaArray.GetString(i), tableArray.GetString(i), tableType)); } } + if (isNative) + { + result.Sort((left, right) => + { + int order = StringComparer.Ordinal.Compare(left.tableType, right.tableType); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.catalog, right.catalog); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.schema, right.schema); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.table, right.table); + }); + } return result; } @@ -901,10 +928,11 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - List batches; + List<(RecordBatch Batch, bool IsNative)> batches; try { - batches = await ExecuteShowColumnsAsync(catalogPattern, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false); + batches = await ExecuteNativeShowColumnsAsync( + catalogPattern, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -915,24 +943,24 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter var tablePositions = new Dictionary(); - foreach (var batch in batches) + foreach (var (batch, isNative) in batches) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableNameArray = TryGetColumn(batch, "tableName"); - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); + var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); + var tableNameArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); + var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); if (catalogArray == null || schemaArray == null || tableNameArray == null || colNameArray == null || columnTypeArray == null) continue; for (int i = 0; i < batch.Length; i++) { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableNameArray.IsNull(i) || + if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableNameArray.IsNull(i) || colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - string cat = catalogArray.GetString(i); + string cat = catalogArray.IsNull(i) ? catalogPattern ?? "" : catalogArray.GetString(i); string sch = schemaArray.GetString(i); string tbl = tableNameArray.GetString(i); string colName = colNameArray.GetString(i); @@ -943,10 +971,15 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter string tableKey = $"{cat}.{sch}.{tbl}"; if (!tablePositions.ContainsKey(tableKey)) tablePositions[tableKey] = 1; - int position = tablePositions[tableKey]++; + int position = isNative + ? checked((int)(NativeMetadataResultBuilder.ReadInteger(batch.Column(16), i) ?? 0) + 1) + : tablePositions[tableKey]++; - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + bool nullable = isNative + ? isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("NO", StringComparison.OrdinalIgnoreCase) + : isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); if (catalogMap.TryGetValue(cat, out var schemaMap) && schemaMap.TryGetValue(sch, out var tableMap) @@ -972,13 +1005,25 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter } internal async Task> ExecuteMetadataSqlAsync(string sql, CancellationToken cancellationToken = default) + { + var (batches, _) = await ExecuteMetadataSqlCoreAsync(sql, null, cancellationToken).ConfigureAwait(false); + return batches; + } + + internal Task<(List Batches, bool IsNative)> ExecuteNativeMetadataSqlAsync( + string sql, MetadataOperation operation, CancellationToken cancellationToken) + => ExecuteMetadataSqlCoreAsync(sql, operation, cancellationToken); + + private async Task<(List Batches, bool IsNative)> ExecuteMetadataSqlCoreAsync( + string sql, MetadataOperation? operation, CancellationToken cancellationToken) { var batches = new List(); using var stmt = (StatementExecutionStatement)CreateStatement(); stmt.SqlQuery = sql; - var result = await stmt.ExecuteQueryAsync(cancellationToken, isMetadataExecution: true).ConfigureAwait(false); + var result = await stmt.ExecuteQueryAsync( + cancellationToken, isMetadataExecution: true, metadataOperation: operation).ConfigureAwait(false); using var stream = result.Stream; - if (stream == null) return batches; + if (stream == null) return (batches, stmt.IsNativeMetadataResult); while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -986,7 +1031,7 @@ internal async Task> ExecuteMetadataSqlAsync(string sql, Cance if (batch == null) break; batches.Add(batch); } - return batches; + return (batches, stmt.IsNativeMetadataResult); } internal List ExecuteMetadataSql(string sql, CancellationToken cancellationToken = default) @@ -994,26 +1039,24 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc return ExecuteMetadataSqlAsync(sql, cancellationToken).GetAwaiter().GetResult(); } - /// - /// Executes a SHOW COLUMNS command. When catalog is null, iterates over all catalogs - /// since SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend. - /// - internal async Task> ExecuteShowColumnsAsync( + // SHOW COLUMNS IN ALL CATALOGS is unavailable, so fan out over catalogs. + internal async Task> ExecuteNativeShowColumnsAsync( string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, CancellationToken cancellationToken) { + var results = new List<(RecordBatch, bool)>(); if (catalog != null) { string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); - return await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); + results.AddRange(batches.Select(batch => (batch, isNative))); + return results; } - // SHOW COLUMNS IN ALL CATALOGS is not supported — iterate over each catalog. - // TODO: Remove this fallback when the backend supports SHOW COLUMNS IN ALL CATALOGS. - var allBatches = new List(); string catalogsSql = new ShowCatalogsCommand(null).Build(); - var catalogBatches = await ExecuteMetadataSqlAsync(catalogsSql, cancellationToken).ConfigureAwait(false); - + var (catalogBatches, _) = await ExecuteNativeMetadataSqlAsync( + catalogsSql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); foreach (var batch in catalogBatches) { var catalogArray = batch.Column(0) as StringArray; @@ -1021,20 +1064,21 @@ internal async Task> ExecuteShowColumnsAsync( for (int i = 0; i < catalogArray.Length; i++) { if (catalogArray.IsNull(i)) continue; - string cat = catalogArray.GetString(i); - string sql = new ShowColumnsCommand(cat, schemaPattern, tablePattern, columnPattern).Build(); + string sql = new ShowColumnsCommand( + catalogArray.GetString(i), schemaPattern, tablePattern, columnPattern).Build(); try { - var batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - allBatches.AddRange(batches); + var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); + results.AddRange(batches.Select(columns => (columns, isNative))); } - catch + catch (Exception) when (!cancellationToken.IsCancellationRequested) { - // Skip catalogs we can't access (permission errors) + // Skip catalogs that cannot be queried. } } } - return allBatches; + return results; } internal bool EnablePKFK => _enablePKFK; diff --git a/csharp/src/StatementExecution/StatementExecutionModels.cs b/csharp/src/StatementExecution/StatementExecutionModels.cs index a2276a7a..f812fe34 100644 --- a/csharp/src/StatementExecution/StatementExecutionModels.cs +++ b/csharp/src/StatementExecution/StatementExecutionModels.cs @@ -19,6 +19,16 @@ namespace AdbcDrivers.Databricks.StatementExecution { + internal enum MetadataOperation + { + GetCatalogs, + GetSchemas, + GetTables, + GetColumns, + GetPrimaryKeys, + GetCrossReference, + } + // ============================================================================ // Session Management Models // ============================================================================ @@ -185,6 +195,9 @@ public class ExecuteStatementRequest [JsonIgnore] public bool IsMetadata { get; set; } + [JsonIgnore] + internal MetadataOperation? MetadataOperation { get; set; } + /// /// Maximum number of rows to return. /// @@ -400,6 +413,9 @@ public class ResultManifest /// [JsonPropertyName("is_volume_operation")] public bool? IsVolumeOperation { get; set; } + + [JsonPropertyName("is_native_metadata_result")] + public bool? IsNativeMetadataResult { get; set; } } /// diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 10542a3a..0aa02d9e 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -79,6 +79,7 @@ internal class StatementExecutionStatement : TracingStatement // in-hand and the statement is closed server-side, so Dispose must NOT re-close it. private bool _statementClosedByServer; private string? _sqlQuery; + internal bool IsNativeMetadataResult { get; private set; } // Marks a driver-internal statement (e.g. the USE CATALOG issued by EnsureCatalogScopedAsync). // Mirrors DatabricksStatement.IsInternalCall on Thrift: it prevents catalog-scoping recursion @@ -305,9 +306,13 @@ public override QueryResult ExecuteQuery() /// Cancellation token. /// When true, adds the x-databricks-sea-can-run-fully-sync /// header for optimized metadata query execution on the server. - public async Task ExecuteQueryAsync( + public Task ExecuteQueryAsync( CancellationToken cancellationToken = default, bool isMetadataExecution = false) + => ExecuteQueryAsync(cancellationToken, isMetadataExecution, null); + + internal async Task ExecuteQueryAsync( + CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation) { if (_isMetadataCommand) { @@ -324,7 +329,7 @@ public async Task ExecuteQueryAsync( lock (_cancelLock) { _executeCts = cts; } try { - return await ExecuteQueryInternalAsync(cts.Token, isMetadataExecution).ConfigureAwait(false); + return await ExecuteQueryInternalAsync(cts.Token, isMetadataExecution, metadataOperation).ConfigureAwait(false); } catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) { @@ -386,8 +391,10 @@ private async Task EnsureCatalogScopedAsync(CancellationToken cancellationToken) _connection.UpdateCurrentCatalog(catalog); } - private async Task ExecuteQueryInternalAsync(CancellationToken cancellationToken, bool isMetadataExecution) + private async Task ExecuteQueryInternalAsync( + CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation = null) { + IsNativeMetadataResult = false; // If the caller explicitly scoped this statement to a catalog, set the session's // current catalog first via USE CATALOG so a 2-level `schema`.`table` name // resolves. The server rejects catalog+session_id together, and SEA always uses a @@ -410,6 +417,7 @@ private async Task ExecuteQueryInternalAsync(CancellationToken canc WaitTimeout = _waitTimeout, OnWaitTimeout = "CONTINUE", IsMetadata = isMetadataExecution, + MetadataOperation = metadataOperation, QueryTags = ParseQueryTags(_queryTags) }; @@ -462,6 +470,8 @@ private async Task ExecuteQueryInternalAsync(CancellationToken canc _statementClosedByServer = true; } + IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult == true; + // Check for truncated results warning if (response.Manifest?.Truncated == true) { @@ -1086,7 +1096,7 @@ private static async Task>> FetchAllChunkSegmentsAsync /// catalog for SHOW commands when not querying all catalogs). /// // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, the - // ExecuteShowColumnsAsync iterate-all-catalogs fallback can be removed. + // ExecuteNativeShowColumnsAsync catalog fanout can be removed. private string? EffectiveCatalog { get @@ -1278,7 +1288,11 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT // matching Thrift behavior (Thrift RPC has no catalog filter for GetCatalogs). string sql = new ShowCatalogsCommand(null).Build(); activity?.SetTag("sql_query", sql); - var batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var (batches, isNative) = await _connection.ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); + if (isNative) + return NativeMetadataResultBuilder.Build( + batches, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); var tableCatBuilder = new StringArray.Builder(); int count = 0; @@ -1329,7 +1343,13 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo List batches; try { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); + batches = nativeResult.Batches; + if (nativeResult.IsNative) + return NativeMetadataResultBuilder.Build( + batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, + requestedCatalog: catalog); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -1413,7 +1433,16 @@ private async Task GetTablesAsync(CancellationToken cancellationTok List batches; try { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); + batches = nativeResult.Batches; + if (nativeResult.IsNative) + { + var requestedTypes = _metadataTableTypes?.Split(',').Select(type => type.Trim()).ToArray(); + return NativeMetadataResultBuilder.Build( + batches, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, + requestedCatalog: catalog, tableTypes: requestedTypes); + } } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -1504,10 +1533,10 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - List batches; + List<(RecordBatch Batch, bool IsNative)> columnBatches; try { - batches = await _connection.ExecuteShowColumnsAsync( + columnBatches = await _connection.ExecuteNativeShowColumnsAsync( catalog, EscapePatternWildcardsInName(_metadataSchemaName), EscapePatternWildcardsInName(_metadataTableName), @@ -1523,26 +1552,34 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo System.Array.Empty<(string, string, string, TableInfo)>()); } + if (columnBatches.Count > 0 && columnBatches.All(result => result.IsNative)) + { + return NativeMetadataResultBuilder.Build( + columnBatches.Select(result => result.Batch).ToList(), + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, + requestedCatalog: catalog); + } + var tableInfos = new Dictionary(); - foreach (var batch in batches) + foreach (var (batch, isNative) in columnBatches) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); + var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); + var tableArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); + var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); if (catalogArray == null || schemaArray == null || tableArray == null || colNameArray == null || columnTypeArray == null) continue; for (int i = 0; i < batch.Length; i++) { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i) || + if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableArray.IsNull(i) || colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - string cat = catalogArray.GetString(i); + string cat = catalogArray.IsNull(i) ? catalog ?? "" : catalogArray.GetString(i); string sch = schemaArray.GetString(i); string tbl = tableArray.GetString(i); string key = $"{cat}.{sch}.{tbl}"; @@ -1552,13 +1589,18 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo var entry = tableInfos[key]; bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); + !isNullableArray.GetString(i).Equals( + isNative ? "NO" : "false", StringComparison.OrdinalIgnoreCase); + + int position = isNative + ? checked((int)(NativeMetadataResultBuilder.ReadInteger(batch.Column(16), i) ?? 0) + 1) + : entry.info.ColumnName.Count; ColumnMetadataHelper.PopulateTableInfoFromTypeName( entry.info, colNameArray.GetString(i), columnTypeArray.GetString(i), - entry.info.ColumnName.Count, + position, nullable); } } @@ -1804,7 +1846,15 @@ private async Task GetPrimaryKeysAsyncNoThrow( List batches; try { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); + batches = nativeResult.Batches; + if (nativeResult.IsNative) + { + using var emptyStream = MetadataSchemaFactory.CreateEmptyPrimaryKeysResult().Stream; + return NativeMetadataResultBuilder.Build( + batches, emptyStream!.Schema, MetadataOperation.GetPrimaryKeys); + } } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -1926,7 +1976,16 @@ private async Task GetCrossReferenceAsyncNoThrow( List batches; try { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); + batches = nativeResult.Batches; + if (nativeResult.IsNative) + { + using var emptyStream = MetadataSchemaFactory.CreateEmptyCrossReferenceResult().Stream; + return NativeMetadataResultBuilder.Build( + batches, emptyStream!.Schema, MetadataOperation.GetCrossReference, + parentCatalog: pkCatalog, parentSchema: pkSchema, parentTable: pkTable); + } } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs new file mode 100644 index 00000000..d2f3bdd7 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Types; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class NativeMetadataResultBuilderTests + { + [Fact] + public async Task GetColumns_CastsIntegersAndMakesOrdinalOneBased() + { + var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); + Assert.Equal(24, target.FieldsList.Count); + var fields = target.FieldsList.Take(23).Select(field => + field.Name == "ORDINAL_POSITION" + ? new Field(field.Name, Int64Type.Default, true) + : field).ToArray(); + var source = new Schema(fields, null); + var arrays = new List(); + foreach (var field in fields) + { + if (field.Name == "ORDINAL_POSITION") + arrays.Add(new Int64Array.Builder().Append(0).Build()); + else if (field.DataType.TypeId == ArrowTypeId.String) + arrays.Add(new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? "DECIMAL(10,2)" : "value").Build()); + else if (field.DataType.TypeId == ArrowTypeId.Int8) + arrays.Add(new Int8Array.Builder().Append(0).Build()); + else if (field.DataType.TypeId == ArrowTypeId.Int16) + arrays.Add(new Int16Array.Builder().Append(0).Build()); + else + arrays.Add(new Int32Array.Builder().Append(0).Build()); + } + + using var nativeBatch = new RecordBatch(source, arrays.ToArray(), 1); + var result = NativeMetadataResultBuilder.Build( + new[] { nativeBatch }, target, MetadataOperation.GetColumns); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + Assert.Equal(1, ((Int32Array)batch.Column(16)).GetValue(0)); + Assert.Equal("DECIMAL", ((StringArray)batch.Column(23)).GetString(0)); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs index dfe58cea..34d571fb 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs @@ -238,6 +238,55 @@ public async Task DeleteSessionAsync_WithHttpError_ThrowsDatabricksException() #region ExecuteStatementAsync Tests + [Fact] + public async Task ExecuteStatementAsync_NativeMetadata_SendsHeadersAndReadsManifest() + { + var request = new ExecuteStatementRequest + { + Statement = "SHOW TABLES IN CATALOG `main`", + WarehouseId = "warehouse-123", + IsMetadata = true, + MetadataOperation = MetadataOperation.GetTables, + }; + var responseJson = JsonSerializer.Serialize(new + { + statement_id = "stmt-native", + status = new { state = "SUCCEEDED" }, + manifest = new { is_native_metadata_result = true }, + }); + HttpRequestMessage? capturedRequest = null; + string? capturedContent = null; + SetupMockResponseWithCapture(HttpStatusCode.OK, responseJson, + req => capturedRequest = req, content => capturedContent = content); + + var response = await new StatementExecutionClient(_httpClient, _testHost) + .ExecuteStatementAsync(request, CancellationToken.None); + + Assert.Equal(true, response.Manifest?.IsNativeMetadataResult); + Assert.Equal("true", Assert.Single(capturedRequest!.Headers.GetValues("x-databricks-sea-can-run-fully-sync"))); + Assert.Equal("GetTables", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-metadata-operation-type"))); + Assert.Equal("true", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-require-thrift-native-metadata"))); + Assert.DoesNotContain("metadataOperation", capturedContent); + Assert.DoesNotContain("is_native_metadata_result", capturedContent); + } + + [Fact] + public async Task ExecuteStatementAsync_MetadataWithoutOperation_OmitsNativeHeaders() + { + var request = new ExecuteStatementRequest { Statement = "DESC TABLE t", IsMetadata = true }; + HttpRequestMessage? capturedRequest = null; + SetupMockResponseWithCapture(HttpStatusCode.OK, + "{\"statement_id\":\"stmt-1\",\"status\":{\"state\":\"SUCCEEDED\"}}", + req => capturedRequest = req, _ => { }); + + await new StatementExecutionClient(_httpClient, _testHost) + .ExecuteStatementAsync(request, CancellationToken.None); + + Assert.True(capturedRequest!.Headers.Contains("x-databricks-sea-can-run-fully-sync")); + Assert.False(capturedRequest.Headers.Contains("x-databricks-metadata-operation-type")); + Assert.False(capturedRequest.Headers.Contains("x-databricks-require-thrift-native-metadata")); + } + [Fact] public async Task ExecuteStatementAsync_WithValidRequest_ReturnsResponse() { diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs index e94e98f3..ad0c781e 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs @@ -113,6 +113,74 @@ private static HttpClient HttpClientReturningShowTables() return new HttpClient(handler.Object); } + private static HttpClient HttpClientReturningNativeTables(List operations) + { + static StringArray ThreeNulls() + { + var builder = new StringArray.Builder(); + builder.AppendNull(); + builder.AppendNull(); + builder.AppendNull(); + return builder.Build(); + } + + string[] names = + { + "TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS", + "TYPE_CAT", "TYPE_SCHEM", "TYPE_NAME", "SELF_REFERENCING_COL_NAME", "REF_GENERATION", + }; + var schema = new Schema(names.Select(name => new Field(name, StringType.Default, true)), null); + var arrays = new IArrowArray[] + { + new StringArray.Builder().Append("main").Append("main").Append("other").Build(), + new StringArray.Builder().Append("default").Append("default").Append("default").Build(), + new StringArray.Builder().Append("my_table").Append("my_view").Append("other_view").Build(), + new StringArray.Builder().Append("TABLE").Append("VIEW").Append("VIEW").Build(), + new StringArray.Builder().Append("table remark").Append("view remark").Append("other remark").Build(), + ThreeNulls(), ThreeNulls(), ThreeNulls(), ThreeNulls(), ThreeNulls(), + }; + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, arrays, 3)); + writer.WriteEnd(); + } + var executeBody = JsonSerializer.Serialize(new + { + statement_id = "stmt-native-tables", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = true, + total_row_count = 3, + schema = new + { + column_count = names.Length, + columns = names.Select((name, position) => new + { + name, position, type_name = "STRING", type_text = "STRING", + }).ToArray(), + }, + }, + result = new { attachment = raw.ToArray() }, + }); + var sessionBody = JsonSerializer.Serialize(new { session_id = "session-1" }); + var handler = new Mock(); + handler.Protected() + .Setup>( + "SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage req, CancellationToken _) => + { + if (req.RequestUri?.AbsolutePath.EndsWith("/api/2.0/sql/sessions") == true) + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(sessionBody) }; + if (req.Method != HttpMethod.Post) + return new HttpResponseMessage(HttpStatusCode.OK); + operations.Add(req.Headers.GetValues("x-databricks-metadata-operation-type").Single()); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(executeBody) }; + }); + return new HttpClient(handler.Object); + } + private static StatementExecutionConnection CreateConnection(HttpClient http) { var properties = new Dictionary @@ -173,6 +241,21 @@ public async Task GetTables_SpecificTableType_ReturnsOnlyThatType() Assert.Equal("TABLE", rows[0].tableType); } + [Fact] + public async Task GetTables_NativeResult_UsesThriftColumns() + { + var operations = new List(); + using var http = HttpClientReturningNativeTables(operations); + using var connection = CreateConnection(http); + + var rows = await ListTablesAsync(connection, new[] { "VIEW" }); + + Assert.Single(rows); + Assert.Equal("my_view", rows[0].table); + Assert.Equal("VIEW", rows[0].tableType); + Assert.Equal(new[] { "GetTables" }, operations); + } + // ─── is_metadata_command path (StatementExecutionStatement.GetTablesAsync) ────── // The metadata-command shim receives tableTypes as a pre-joined string option // (adbc.get_metadata.target_table_types). An empty string means "match none" @@ -231,5 +314,17 @@ public async Task MetadataCommand_SpecificTableType_ReturnsOnlyThatType() Assert.Single(types); Assert.Equal("TABLE", types[0]); } + + [Fact] + public async Task MetadataCommand_NativeTables_FiltersByType() + { + var operations = new List(); + using var http = HttpClientReturningNativeTables(operations); + + var types = await MetadataCommandTableTypes(http, "TABLE"); + + Assert.Equal(new[] { "TABLE" }, types); + Assert.Equal(new[] { "GetTables" }, operations); + } } } From 29389366da6fe1e5e40d07a396738b27e5ef030f Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 18:56:21 +0000 Subject: [PATCH 02/22] fix(csharp): align native metadata with existing ADBC results Signed-off-by: Vu Anh Phung --- csharp/README.md | 1 + csharp/src/DatabricksParameters.cs | 7 + csharp/src/FlatColumnsResultBuilder.cs | 3 +- .../ColumnMetadataHelper.cs | 5 +- .../NativeMetadataColumns.cs | 59 +++++ .../NativeMetadataResultBuilder.cs | 83 +++++--- .../StatementExecutionClient.cs | 11 +- .../StatementExecutionConnection.cs | 201 ++++++++++++------ .../StatementExecutionStatement.cs | 98 ++++++--- .../NativeMetadataResultBuilderTests.cs | 92 +++++++- .../StatementExecutionClientTests.cs | 8 +- ...entExecutionMetadataObjectNotFoundTests.cs | 40 ++++ .../StatementExecutionNativeCatalogTests.cs | 169 +++++++++++++++ ...atementExecutionPollingFailedStateTests.cs | 35 +++ 14 files changed, 675 insertions(+), 137 deletions(-) create mode 100644 csharp/src/StatementExecution/NativeMetadataColumns.cs create mode 100644 csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs diff --git a/csharp/README.md b/csharp/README.md index 315866ce..b424d166 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -218,6 +218,7 @@ The Databricks driver supports two protocols: **Thrift** (default, HiveServer2) | `adbc.databricks.max_bytes_per_fetch_request` | Max bytes per fetch request (supports B, KB, MB, GB) | `400MB` | Not Supported | | `adbc.databricks.apply_ssp_with_queries` | Apply server-side properties with queries | `false` | Not Supported | | `adbc.databricks.enable_multiple_catalog_support` | Support multiple catalogs | `true` | `true` | +| `adbc.databricks.enable_thrift_native_metadata` | Require native metadata results for SEA metadata queries; set `false` to use SHOW results | Not Supported | `true` | | `adbc.databricks.enable_pk_fk` | Enable primary/foreign key metadata | `true` | `true` | | `adbc.databricks.use_desc_table_extended` | Use DESC TABLE EXTENDED when supported | `true` | `true` | | `adbc.databricks.enable_run_async_thrift` | Enable RunAsync flag | `true` | Not Supported (Thrift-only concept) | diff --git a/csharp/src/DatabricksParameters.cs b/csharp/src/DatabricksParameters.cs index 8e56b2db..76aad46a 100644 --- a/csharp/src/DatabricksParameters.cs +++ b/csharp/src/DatabricksParameters.cs @@ -339,6 +339,13 @@ public class DatabricksParameters : SparkParameters [FeatureFlagType(FeatureFlagValueKind.Boolean)] public const string EnableFastMetadataQuery = "adbc.databricks.enable_fast_metadata_query"; + /// + /// Require Thrift-shaped native metadata results on Statement Execution metadata queries. + /// Default value is true. Set false to use SHOW results instead. + /// + [FeatureFlagType(FeatureFlagValueKind.Boolean)] + public const string EnableThriftNativeMetadata = "adbc.databricks.enable_thrift_native_metadata"; + /// /// Whether to enable RunAsync flag in Thrift operation /// Default value is true if not specified. diff --git a/csharp/src/FlatColumnsResultBuilder.cs b/csharp/src/FlatColumnsResultBuilder.cs index 30ae73e0..064593bb 100644 --- a/csharp/src/FlatColumnsResultBuilder.cs +++ b/csharp/src/FlatColumnsResultBuilder.cs @@ -81,7 +81,8 @@ internal static QueryResult BuildFlatColumnsResult( nullableBuilder.Append(info.Nullable[i]); remarksBuilder.Append(""); - columnDefBuilder.AppendNull(); + if (string.IsNullOrEmpty(info.ColumnDefault[i])) columnDefBuilder.AppendNull(); + else columnDefBuilder.Append(info.ColumnDefault[i]); sqlDataTypeBuilder.Append(info.ColType[i]); diff --git a/csharp/src/StatementExecution/ColumnMetadataHelper.cs b/csharp/src/StatementExecution/ColumnMetadataHelper.cs index b3755cb7..93927a88 100644 --- a/csharp/src/StatementExecution/ColumnMetadataHelper.cs +++ b/csharp/src/StatementExecution/ColumnMetadataHelper.cs @@ -242,7 +242,8 @@ internal static void PopulateTableInfoFromTypeName( int ordinalPosition, bool isNullable = true, string? comment = null, - string? columnDefault = null) + string? columnDefault = null, + bool isAutoIncrement = false) { tableInfo.ColumnName.Add(columnName); tableInfo.TypeName.Add(typeName); @@ -254,7 +255,7 @@ internal static void PopulateTableInfoFromTypeName( tableInfo.OrdinalPosition.Add(ordinalPosition); tableInfo.Nullable.Add(isNullable ? (short)1 : (short)0); tableInfo.IsNullable.Add(isNullable ? "YES" : "NO"); - tableInfo.IsAutoIncrement.Add(false); + tableInfo.IsAutoIncrement.Add(isAutoIncrement); tableInfo.ColumnDefault.Add(columnDefault ?? ""); } diff --git a/csharp/src/StatementExecution/NativeMetadataColumns.cs b/csharp/src/StatementExecution/NativeMetadataColumns.cs new file mode 100644 index 00000000..5d3dd4f7 --- /dev/null +++ b/csharp/src/StatementExecution/NativeMetadataColumns.cs @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using AdbcDrivers.HiveServer2; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal sealed class NativeMetadataColumns + { + private readonly Dictionary _columns = new(StringComparer.Ordinal); + + internal NativeMetadataColumns(RecordBatch batch, Schema schema, MetadataOperation operation) + { + int count = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); + if (batch.ColumnCount != count) + throw new DatabricksException($"Invalid native {operation} result: expected {count} columns, found {batch.ColumnCount}"); + + for (int i = 0; i < count; i++) + { + Field expected = schema.FieldsList[i]; + int index = batch.Schema.GetFieldIndex(expected.Name); + if (index < 0) + throw new DatabricksException($"Invalid native {operation} result: missing {expected.Name}"); + + IArrowArray array = batch.Column(index); + bool valid = expected.DataType.TypeId == ArrowTypeId.String + ? array is StringArray + : IsInteger(expected.DataType.TypeId) && IsInteger(batch.Schema.FieldsList[index].DataType.TypeId) + && array is Int8Array or Int16Array or Int32Array or Int64Array; + if (!valid || !_columns.TryAdd(expected.Name, array)) + throw new DatabricksException($"Invalid native {operation} result: unexpected type or duplicate {expected.Name}"); + } + } + + internal string? String(string name, int row) => NativeMetadataResultBuilder.ReadString(_columns[name], row); + + internal long? Integer(string name, int row) => NativeMetadataResultBuilder.ReadInteger(_columns[name], row); + + private static bool IsInteger(ArrowTypeId type) => type is + ArrowTypeId.Int8 or ArrowTypeId.Int16 or ArrowTypeId.Int32 or ArrowTypeId.Int64; + } +} diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 01ab2a07..9b790498 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -30,32 +30,36 @@ internal static class NativeMetadataResultBuilder internal static QueryResult Build( IReadOnlyList batches, Schema schema, MetadataOperation operation, string? requestedCatalog = null, IReadOnlyCollection? tableTypes = null, - string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null, + IReadOnlyList? sourceCatalogs = null) { // C# exposes BASE_TYPE_NAME after the 23 Thrift GetColumns fields. int sourceColumns = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); - var rows = new List<(RecordBatch Batch, int Index)>(); - foreach (var batch in batches) + if (sourceCatalogs != null && sourceCatalogs.Count != batches.Count) + throw new ArgumentException("Each native batch must have a source catalog", nameof(sourceCatalogs)); + var rows = new List<(NativeMetadataColumns Columns, int Index, string? Catalog)>(); + for (int batchIndex = 0; batchIndex < batches.Count; batchIndex++) { - if (batch.Schema.FieldsList.Count != sourceColumns) - throw new DatabricksException($"Invalid native {operation} result: expected {sourceColumns} columns, found {batch.Schema.FieldsList.Count}"); + var batch = batches[batchIndex]; + var columns = new NativeMetadataColumns(batch, schema, operation); + string? sourceCatalog = sourceCatalogs?[batchIndex] ?? requestedCatalog; for (int row = 0; row < batch.Length; row++) { if (operation == MetadataOperation.GetTables && requestedCatalog != null && - !string.Equals(requestedCatalog, ReadString(batch.Column(0), row), StringComparison.OrdinalIgnoreCase)) + !string.Equals(requestedCatalog, columns.String("TABLE_CAT", row) ?? sourceCatalog, StringComparison.OrdinalIgnoreCase)) continue; if (operation == MetadataOperation.GetTables && tableTypes != null && - !tableTypes.Contains(ReadString(batch.Column(3), row) ?? string.Empty)) + !tableTypes.Contains(DefaultTableType(columns.String("TABLE_TYPE", row)))) continue; if (operation == MetadataOperation.GetCrossReference && - (!Matches(parentCatalog, ReadString(batch.Column(0), row)) || - !Matches(parentSchema, ReadString(batch.Column(1), row)) || - !Matches(parentTable, ReadString(batch.Column(2), row)))) + (!Matches(parentCatalog, columns.String(schema.FieldsList[0].Name, row)) || + !Matches(parentSchema, columns.String(schema.FieldsList[1].Name, row)) || + !Matches(parentTable, columns.String(schema.FieldsList[2].Name, row)))) continue; - rows.Add((batch, row)); + rows.Add((columns, row, sourceCatalog)); } } @@ -66,9 +70,20 @@ internal static QueryResult Build( { foreach (int column in sortColumns) { + string? leftValue = left.Columns.String(schema.FieldsList[column].Name, left.Index); + string? rightValue = right.Columns.String(schema.FieldsList[column].Name, right.Index); + if (column == 3) + { + leftValue = DefaultTableType(leftValue); + rightValue = DefaultTableType(rightValue); + } + if (column == 0) + { + leftValue ??= left.Catalog; + rightValue ??= right.Catalog; + } int result = StringComparer.Ordinal.Compare( - ReadString(left.Batch.Column(column), left.Index), - ReadString(right.Batch.Column(column), right.Index)); + leftValue, rightValue); if (result != 0) return result; } return 0; @@ -82,18 +97,20 @@ internal static QueryResult Build( { case ArrowTypeId.String: var strings = new StringArray.Builder(); - foreach (var (batch, row) in rows) + foreach (var (columns, row, sourceCatalog) in rows) { string? typeName = column == sourceColumns && operation == MetadataOperation.GetColumns - ? ReadString(batch.Column(5), row) + ? columns.String("TYPE_NAME", row) : null; string? value = column == sourceColumns && operation == MetadataOperation.GetColumns ? typeName == null ? null : ColumnMetadataHelper.GetBaseTypeName(typeName) - : ReadString(batch.Column(column), row); + : columns.String(schema.FieldsList[column].Name, row); if (column == 1 && operation == MetadataOperation.GetSchemas) - value ??= requestedCatalog; + value ??= sourceCatalog ?? ""; if (column == 0 && (operation == MetadataOperation.GetTables || operation == MetadataOperation.GetColumns)) - value ??= requestedCatalog; + value ??= sourceCatalog ?? ""; + if (column == 3 && operation == MetadataOperation.GetTables) + value = DefaultTableType(value); if (value == null) strings.AppendNull(); else strings.Append(value); } arrays.Add(strings.Build()); @@ -101,9 +118,9 @@ internal static QueryResult Build( case ArrowTypeId.Int8: var int8 = new Int8Array.Builder(); - foreach (var (batch, row) in rows) + foreach (var (columns, row, _) in rows) { - long? value = ReadInteger(batch.Column(column), row); + long? value = columns.Integer(schema.FieldsList[column].Name, row); if (value.HasValue) int8.Append(checked((sbyte)value.Value)); else int8.AppendNull(); } arrays.Add(int8.Build()); @@ -111,9 +128,9 @@ internal static QueryResult Build( case ArrowTypeId.Int16: var int16 = new Int16Array.Builder(); - foreach (var (batch, row) in rows) + foreach (var (columns, row, _) in rows) { - long? value = ReadInteger(batch.Column(column), row); + long? value = columns.Integer(schema.FieldsList[column].Name, row); if (value.HasValue) int16.Append(checked((short)value.Value)); else int16.AppendNull(); } arrays.Add(int16.Build()); @@ -121,12 +138,18 @@ internal static QueryResult Build( case ArrowTypeId.Int32: var int32 = new Int32Array.Builder(); - foreach (var (batch, row) in rows) + foreach (var (columns, row, _) in rows) { - long? value = ReadInteger(batch.Column(column), row); - if (value.HasValue && operation == MetadataOperation.GetColumns && - schema.FieldsList[column].Name == "ORDINAL_POSITION") - value = checked(value.Value + 1); + string name = schema.FieldsList[column].Name; + long? value = columns.Integer(name, row); + if (operation == MetadataOperation.GetColumns && (name is "COLUMN_SIZE" or "DECIMAL_DIGITS")) + { + string? typeName = columns.String("TYPE_NAME", row); + if (typeName != null) + value = name == "COLUMN_SIZE" + ? ColumnMetadataHelper.GetColumnSizeDefault(typeName) + : ColumnMetadataHelper.GetDecimalDigitsDefault(typeName); + } if (value.HasValue) int32.Append(checked((int)value.Value)); else int32.AppendNull(); } arrays.Add(int32.Build()); @@ -134,9 +157,9 @@ internal static QueryResult Build( case ArrowTypeId.Int64: var int64 = new Int64Array.Builder(); - foreach (var (batch, row) in rows) + foreach (var (columns, row, _) in rows) { - long? value = ReadInteger(batch.Column(column), row); + long? value = columns.Integer(schema.FieldsList[column].Name, row); if (value.HasValue) int64.Append(value.Value); else int64.AppendNull(); } arrays.Add(int64.Build()); @@ -173,5 +196,7 @@ internal static QueryResult Build( private static bool Matches(string? requested, string? actual) => requested == null || (actual != null && string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase)); + + internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value; } } diff --git a/csharp/src/StatementExecution/StatementExecutionClient.cs b/csharp/src/StatementExecution/StatementExecutionClient.cs index 9b1affd7..96ece3c5 100644 --- a/csharp/src/StatementExecution/StatementExecutionClient.cs +++ b/csharp/src/StatementExecution/StatementExecutionClient.cs @@ -229,7 +229,16 @@ public async Task ExecuteStatementAsync( if (request.IsMetadata && request.MetadataOperation is MetadataOperation operation) { - httpRequest.Headers.TryAddWithoutValidation("x-databricks-metadata-operation-type", operation.ToString()); + httpRequest.Headers.TryAddWithoutValidation("x-databricks-metadata-operation-type", operation switch + { + MetadataOperation.GetCatalogs => "GetCatalogs", + MetadataOperation.GetSchemas => "GetSchemas", + MetadataOperation.GetTables => "GetTables", + MetadataOperation.GetColumns => "GetColumns", + MetadataOperation.GetPrimaryKeys => "GetPrimaryKeys", + MetadataOperation.GetCrossReference => "GetCrossReference", + _ => throw new ArgumentOutOfRangeException(nameof(operation), operation, null), + }); httpRequest.Headers.TryAddWithoutValidation("x-databricks-require-thrift-native-metadata", "true"); } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 24c566b1..a7370628 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -18,6 +18,8 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; +using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.Http; @@ -70,6 +72,7 @@ internal class StatementExecutionConnection : TracingConnection, IGetObjectsData private bool _scopeCurrentCatalog; private bool _useDescTableExtended; private bool _enableFastMetadataQuery; + private bool _enableThriftNativeMetadata; private bool _applySSPWithQueries; // Connection bring-up timeout (PECO-3059). Mirrors the Thrift path's @@ -326,6 +329,7 @@ private void ValidateProperties() _scopeCurrentCatalog = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.ScopeCurrentCatalog, false); _useDescTableExtended = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.UseDescTableExtended, true); _enableFastMetadataQuery = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.EnableFastMetadataQuery, false); + _enableThriftNativeMetadata = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.EnableThriftNativeMetadata, true); // When true, SSPs (adbc.databricks.ssp_*) are applied via post-open SET statements // rather than CreateSession.session_confs — mirrors Thrift's behavior so callers // who depend on the SET-statement path (e.g., for audit visibility or for SSPs @@ -713,7 +717,7 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string // which sends catalog as-is to the server. ExecuteNativeShowColumnsAsync // handles null by iterating all catalogs. string? resolvedCatalog = DatabricksConnection.HandleSparkCatalog(catalog); - List<(RecordBatch Batch, bool IsNative)> batches; + List<(RecordBatch Batch, bool IsNative, string? Catalog)> batches; try { batches = ExecuteNativeShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) @@ -728,23 +732,27 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string } var fields = new List(); - foreach (var (batch, isNative) in batches) + foreach (var (batch, isNative, _) in batches) { - var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); + var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; + var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - if (colNameArray == null || columnTypeArray == null) continue; + if (!isNative && (colNameArray == null || columnTypeArray == null)) continue; for (int i = 0; i < batch.Length; i++) { - if (colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; + if (isNative + ? native!.String("COLUMN_NAME", i) == null || native.String("TYPE_NAME", i) == null + : colNameArray!.IsNull(i) || columnTypeArray!.IsNull(i)) continue; - string colName = colNameArray.GetString(i); - string colType = columnTypeArray.GetString(i); - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals( - isNative ? "NO" : "false", StringComparison.OrdinalIgnoreCase); + string colName = isNative ? native!.String("COLUMN_NAME", i)! : colNameArray!.GetString(i); + string colType = isNative ? native!.String("TYPE_NAME", i)! : columnTypeArray!.GetString(i); + bool nullable = isNative + ? native!.Integer("NULLABLE", i) == 1 + : isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); short typeCode = ColumnMetadataHelper.GetDataTypeCode(colType); IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, colType, false, null, null); @@ -778,12 +786,15 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin var result = new List(); foreach (var batch in batches) { - var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalog"); - if (catalogArray == null) continue; + var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) : null; + var catalogArray = isNative ? null : TryGetColumn(batch, "catalog"); + if (!isNative && catalogArray == null) continue; for (int i = 0; i < batch.Length; i++) { - if (!catalogArray.IsNull(i)) - result.Add(catalogArray.GetString(i)); + string? name = isNative ? native!.String("TABLE_CAT", i) + : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); + if (name != null && (!isNative || MatchesCatalogPattern(catalogPattern, name))) + result.Add(name); } } return result; @@ -821,8 +832,14 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin if (isNative) { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; + var native = new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); + for (int i = 0; i < batch.Length; i++) + { + string? name = native.String("TABLE_SCHEM", i); + if (name != null) + result.Add((native.String("TABLE_CATALOG", i) ?? catalogPattern ?? "", name)); + } + continue; } else if (showSchemasInAllCatalogs) { @@ -876,27 +893,30 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin var result = new List<(string catalog, string schema, string table, string tableType)>(); foreach (var batch in batches) { - var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); - var tableArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); - var tableTypeArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "tableType"); + var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables) : null; + var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); + var tableArray = isNative ? null : TryGetColumn(batch, "tableName"); + var tableTypeArray = isNative ? null : TryGetColumn(batch, "tableType"); - if (catalogArray == null || schemaArray == null || tableArray == null) continue; + if (!isNative && (catalogArray == null || schemaArray == null || tableArray == null)) continue; for (int i = 0; i < batch.Length; i++) { - if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; + string? cat = isNative ? native!.String("TABLE_CAT", i) ?? catalogPattern + : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); + string? sch = isNative ? native!.String("TABLE_SCHEM", i) + : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); + string? tbl = isNative ? native!.String("TABLE_NAME", i) + : tableArray!.IsNull(i) ? null : tableArray.GetString(i); + if ((!isNative && cat == null) || sch == null || tbl == null) continue; + cat ??= ""; if (isNative && catalogPattern != null && - !string.Equals(catalogPattern, catalogArray.IsNull(i) ? null : catalogArray.GetString(i), - StringComparison.OrdinalIgnoreCase)) continue; + !string.Equals(catalogPattern, cat, StringComparison.OrdinalIgnoreCase)) continue; - string tableType = "TABLE"; - if (tableTypeArray != null && !tableTypeArray.IsNull(i)) - { - string serverType = tableTypeArray.GetString(i); - if (!string.IsNullOrEmpty(serverType)) - tableType = serverType; - } + string tableType = NativeMetadataResultBuilder.DefaultTableType(isNative + ? native!.String("TABLE_TYPE", i) + : tableTypeArray == null || tableTypeArray.IsNull(i) ? null : tableTypeArray.GetString(i)); // Empty (non-null) tableTypes filters to NO types (zero rows), // matching databricks-jdbc. A null tableTypes means "all types" @@ -904,8 +924,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin if (tableTypes != null && !tableTypes.Contains(tableType)) continue; - result.Add((catalogArray.IsNull(i) ? catalogPattern ?? "" : catalogArray.GetString(i), schemaArray.GetString(i), - tableArray.GetString(i), tableType)); + result.Add((cat, sch, tbl, tableType)); } } if (isNative) @@ -928,7 +947,7 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - List<(RecordBatch Batch, bool IsNative)> batches; + List<(RecordBatch Batch, bool IsNative, string? Catalog)> batches; try { batches = await ExecuteNativeShowColumnsAsync( @@ -943,28 +962,32 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter var tablePositions = new Dictionary(); - foreach (var (batch, isNative) in batches) + foreach (var (batch, isNative, sourceCatalog) in batches) { - var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); - var tableNameArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); - var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); + var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; + var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); + var tableNameArray = isNative ? null : TryGetColumn(batch, "tableName"); + var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - if (catalogArray == null || schemaArray == null || tableNameArray == null || - colNameArray == null || columnTypeArray == null) continue; + if (!isNative && (catalogArray == null || schemaArray == null || tableNameArray == null || + colNameArray == null || columnTypeArray == null)) continue; for (int i = 0; i < batch.Length; i++) { - if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableNameArray.IsNull(i) || - colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - - string cat = catalogArray.IsNull(i) ? catalogPattern ?? "" : catalogArray.GetString(i); - string sch = schemaArray.GetString(i); - string tbl = tableNameArray.GetString(i); - string colName = colNameArray.GetString(i); - string colType = columnTypeArray.GetString(i); + string? cat = isNative ? native!.String("TABLE_CAT", i) ?? sourceCatalog ?? catalogPattern + : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); + string? sch = isNative ? native!.String("TABLE_SCHEM", i) + : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); + string? tbl = isNative ? native!.String("TABLE_NAME", i) + : tableNameArray!.IsNull(i) ? null : tableNameArray.GetString(i); + string? colName = isNative ? native!.String("COLUMN_NAME", i) + : colNameArray!.IsNull(i) ? null : colNameArray.GetString(i); + string? colType = isNative ? native!.String("TYPE_NAME", i) + : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); + if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; if (string.IsNullOrEmpty(colName)) continue; @@ -972,12 +995,11 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter if (!tablePositions.ContainsKey(tableKey)) tablePositions[tableKey] = 1; int position = isNative - ? checked((int)(NativeMetadataResultBuilder.ReadInteger(batch.Column(16), i) ?? 0) + 1) + ? checked((int)(native!.Integer("ORDINAL_POSITION", i) ?? 0) + 1) : tablePositions[tableKey]++; bool nullable = isNative - ? isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("NO", StringComparison.OrdinalIgnoreCase) + ? native!.Integer("NULLABLE", i) == 1 : isNullableArray == null || isNullableArray.IsNull(i) || !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); @@ -986,7 +1008,9 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter && tableMap.TryGetValue(tbl, out var tableInfo)) { ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfo, colName, colType, position, nullable); + tableInfo, colName, colType, position, nullable, + columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, + isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTOINCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); } } } @@ -1040,47 +1064,84 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc } // SHOW COLUMNS IN ALL CATALOGS is unavailable, so fan out over catalogs. - internal async Task> ExecuteNativeShowColumnsAsync( + internal async Task> ExecuteNativeShowColumnsAsync( string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, CancellationToken cancellationToken) { - var results = new List<(RecordBatch, bool)>(); + var results = new List<(RecordBatch, bool, string?)>(); if (catalog != null) { string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(batch => (batch, isNative))); + results.AddRange(batches.Select(batch => (batch, isNative, catalog))); return results; } string catalogsSql = new ShowCatalogsCommand(null).Build(); - var (catalogBatches, _) = await ExecuteNativeMetadataSqlAsync( + var (catalogBatches, catalogsAreNative) = await ExecuteNativeMetadataSqlAsync( catalogsSql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); foreach (var batch in catalogBatches) { - var catalogArray = batch.Column(0) as StringArray; - if (catalogArray == null) continue; - for (int i = 0; i < catalogArray.Length; i++) + var native = catalogsAreNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) : null; + var catalogArray = catalogsAreNative ? null : TryGetColumn(batch, "catalog"); + if (!catalogsAreNative && catalogArray == null) continue; + for (int i = 0; i < batch.Length; i++) { - if (catalogArray.IsNull(i)) continue; + string? sourceCatalog = catalogsAreNative ? native!.String("TABLE_CAT", i) + : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); + if (sourceCatalog == null) continue; string sql = new ShowColumnsCommand( - catalogArray.GetString(i), schemaPattern, tablePattern, columnPattern).Build(); + sourceCatalog, schemaPattern, tablePattern, columnPattern).Build(); try { var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(columns => (columns, isNative))); + results.AddRange(batches.Select(columns => (columns, isNative, sourceCatalog))); } - catch (Exception) when (!cancellationToken.IsCancellationRequested) + catch (DatabricksException ex) when (!cancellationToken.IsCancellationRequested && + (ex.IsObjectNotFoundException() || IsSqlPermissionDenied(ex))) { - // Skip catalogs that cannot be queried. + System.Diagnostics.Activity.Current?.AddEvent(new System.Diagnostics.ActivityEvent( + "statement.get_columns.catalog_skipped", + tags: new System.Diagnostics.ActivityTagsCollection + { + { "catalog", sourceCatalog }, + { "error", ex.Message }, + })); } } } return results; } + private static bool IsSqlPermissionDenied(DatabricksException ex) + => ex.SqlState == "42501" || ex.Message.IndexOf("SQLSTATE: 42501", StringComparison.OrdinalIgnoreCase) >= 0 + || ex.Message.IndexOf("INSUFFICIENT_PERMISSIONS", StringComparison.OrdinalIgnoreCase) >= 0; + + private static bool MatchesCatalogPattern(string? pattern, string catalog) + { + if (pattern == null) return true; + var regex = new StringBuilder("^"); + bool escaped = false; + foreach (char character in pattern) + { + if (!escaped && character == '\\') + { + escaped = true; + continue; + } + regex.Append(!escaped && character == '%' ? ".*" + : !escaped && character == '_' ? "." + : Regex.Escape(character.ToString())); + escaped = false; + } + if (escaped) regex.Append(Regex.Escape("\\")); + regex.Append('$'); + return Regex.IsMatch(catalog, regex.ToString(), RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + } + internal bool EnablePKFK => _enablePKFK; internal bool EnableMultipleCatalogSupport => _enableMultipleCatalogSupport; @@ -1128,6 +1189,8 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc /// internal bool EnableFastMetadataQuery => _enableFastMetadataQuery; + internal bool EnableThriftNativeMetadata => _enableThriftNativeMetadata; + /// /// Returns the session's default catalog by querying the server via /// SELECT CURRENT_CATALOG(). SEA's CreateSession response, unlike Thrift's diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 0aa02d9e..881e1580 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -417,12 +417,13 @@ private async Task ExecuteQueryInternalAsync( WaitTimeout = _waitTimeout, OnWaitTimeout = "CONTINUE", IsMetadata = isMetadataExecution, - MetadataOperation = metadataOperation, + MetadataOperation = _connection.EnableThriftNativeMetadata ? metadataOperation : null, QueryTags = ParseQueryTags(_queryTags) }; // Execute the statement var response = await _client.ExecuteStatementAsync(request, cancellationToken).ConfigureAwait(false); + bool? initialNativeMetadata = response.Manifest?.IsNativeMetadataResult; _currentStatementId = response.StatementId; // Reset per-execution: statements are reusable, so this must reflect the CURRENT // execution's state, not a prior CLOSED result. Otherwise a later SUCCEEDED (genuinely @@ -470,7 +471,9 @@ private async Task ExecuteQueryInternalAsync( _statementClosedByServer = true; } - IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult == true; + IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult + ?? initialNativeMetadata + ?? HasNativeMetadataSchema(response.Manifest?.Schema, metadataOperation); // Check for truncated results warning if (response.Manifest?.Truncated == true) @@ -514,6 +517,32 @@ private async Task ExecuteQueryInternalAsync( return new QueryResult(rowCount, reader); } + private static bool HasNativeMetadataSchema(ResultSchema? resultSchema, MetadataOperation? operation) + { + if (operation == null || resultSchema?.Columns == null) + return false; + + Schema expected = operation.Value switch + { + MetadataOperation.GetCatalogs => MetadataSchemaFactory.CreateCatalogsSchema(), + MetadataOperation.GetSchemas => MetadataSchemaFactory.CreateSchemasSchema(), + MetadataOperation.GetTables => MetadataSchemaFactory.CreateTablesSchema(), + MetadataOperation.GetColumns => MetadataSchemaFactory.CreateColumnMetadataSchema(), + MetadataOperation.GetPrimaryKeys => MetadataSchemaFactory.CreatePrimaryKeysSchema(), + MetadataOperation.GetCrossReference => MetadataSchemaFactory.CreateCrossReferenceSchema(), + _ => throw new ArgumentOutOfRangeException(nameof(operation)), + }; + int count = expected.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); + if (resultSchema.Columns.Count != count) + return false; + for (int i = 0; i < count; i++) + { + if (!string.Equals(resultSchema.Columns[i].Name, expected.FieldsList[i].Name, StringComparison.Ordinal)) + return false; + } + return true; + } + /// /// Wraps PollUntilCompleteAsync with query timeout enforcement. /// If _queryTimeoutSeconds > 0, cancels the server-side statement and throws on timeout. @@ -1486,7 +1515,8 @@ private async Task GetTablesAsync(CancellationToken cancellationTok for (int i = 0; i < batch.Length; i++) { if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; - string tableType = tableTypeArray != null && !tableTypeArray.IsNull(i) ? tableTypeArray.GetString(i) : "TABLE"; + string tableType = NativeMetadataResultBuilder.DefaultTableType( + tableTypeArray != null && !tableTypeArray.IsNull(i) ? tableTypeArray.GetString(i) : null); if (tableTypeFilter != null && !tableTypeFilter.Contains(tableType)) continue; tableCatBuilder.Append(catalogArray.GetString(i)); tableSchemaBuilder.Append(schemaArray.GetString(i)); @@ -1533,7 +1563,7 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - List<(RecordBatch Batch, bool IsNative)> columnBatches; + List<(RecordBatch Batch, bool IsNative, string? Catalog)> columnBatches; try { columnBatches = await _connection.ExecuteNativeShowColumnsAsync( @@ -1557,51 +1587,61 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return NativeMetadataResultBuilder.Build( columnBatches.Select(result => result.Batch).ToList(), MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, - requestedCatalog: catalog); + requestedCatalog: catalog, + sourceCatalogs: columnBatches.Select(result => result.Catalog).ToList()); } var tableInfos = new Dictionary(); - foreach (var (batch, isNative) in columnBatches) + foreach (var (batch, isNative, sourceCatalog) in columnBatches) { - var catalogArray = isNative ? batch.Column(0) as StringArray : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? batch.Column(1) as StringArray : TryGetColumn(batch, "namespace"); - var tableArray = isNative ? batch.Column(2) as StringArray : TryGetColumn(batch, "tableName"); - var colNameArray = isNative ? batch.Column(3) as StringArray : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? batch.Column(5) as StringArray : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? batch.Column(17) as StringArray : TryGetColumn(batch, "isNullable"); + var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; + var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); + var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); + var tableArray = isNative ? null : TryGetColumn(batch, "tableName"); + var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); + var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); + var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - if (catalogArray == null || schemaArray == null || tableArray == null || - colNameArray == null || columnTypeArray == null) continue; + if (!isNative && (catalogArray == null || schemaArray == null || tableArray == null || + colNameArray == null || columnTypeArray == null)) continue; for (int i = 0; i < batch.Length; i++) { - if ((!isNative && catalogArray.IsNull(i)) || schemaArray.IsNull(i) || tableArray.IsNull(i) || - colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; + string? cat = isNative ? native!.String("TABLE_CAT", i) ?? sourceCatalog ?? catalog + : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); + string? sch = isNative ? native!.String("TABLE_SCHEM", i) + : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); + string? tbl = isNative ? native!.String("TABLE_NAME", i) + : tableArray!.IsNull(i) ? null : tableArray.GetString(i); + string? colName = isNative ? native!.String("COLUMN_NAME", i) + : colNameArray!.IsNull(i) ? null : colNameArray.GetString(i); + string? colType = isNative ? native!.String("TYPE_NAME", i) + : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); + if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; - string cat = catalogArray.IsNull(i) ? catalog ?? "" : catalogArray.GetString(i); - string sch = schemaArray.GetString(i); - string tbl = tableArray.GetString(i); string key = $"{cat}.{sch}.{tbl}"; if (!tableInfos.ContainsKey(key)) tableInfos[key] = (cat, sch, tbl, new TableInfo("TABLE")); var entry = tableInfos[key]; - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals( - isNative ? "NO" : "false", StringComparison.OrdinalIgnoreCase); + bool nullable = isNative ? native!.Integer("NULLABLE", i) == 1 + : isNullableArray == null || isNullableArray.IsNull(i) || + !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); int position = isNative - ? checked((int)(NativeMetadataResultBuilder.ReadInteger(batch.Column(16), i) ?? 0) + 1) + ? checked((int)(native!.Integer("ORDINAL_POSITION", i) ?? 0)) : entry.info.ColumnName.Count; ColumnMetadataHelper.PopulateTableInfoFromTypeName( entry.info, - colNameArray.GetString(i), - columnTypeArray.GetString(i), + colName, + colType, position, - nullable); + nullable, + columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, + isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTOINCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); } } @@ -1851,9 +1891,8 @@ private async Task GetPrimaryKeysAsyncNoThrow( batches = nativeResult.Batches; if (nativeResult.IsNative) { - using var emptyStream = MetadataSchemaFactory.CreateEmptyPrimaryKeysResult().Stream; return NativeMetadataResultBuilder.Build( - batches, emptyStream!.Schema, MetadataOperation.GetPrimaryKeys); + batches, MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys); } } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) @@ -1981,9 +2020,8 @@ private async Task GetCrossReferenceAsyncNoThrow( batches = nativeResult.Batches; if (nativeResult.IsNative) { - using var emptyStream = MetadataSchemaFactory.CreateEmptyCrossReferenceResult().Stream; return NativeMetadataResultBuilder.Build( - batches, emptyStream!.Schema, MetadataOperation.GetCrossReference, + batches, MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, parentCatalog: pkCatalog, parentSchema: pkSchema, parentTable: pkTable); } } diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index d2f3bdd7..e7465758 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -29,7 +29,7 @@ namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution public class NativeMetadataResultBuilderTests { [Fact] - public async Task GetColumns_CastsIntegersAndMakesOrdinalOneBased() + public async Task GetColumns_NormalizesSizeAndScaleAndKeepsFlatOrdinalZeroBased() { var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); Assert.Equal(24, target.FieldsList.Count); @@ -43,6 +43,8 @@ public async Task GetColumns_CastsIntegersAndMakesOrdinalOneBased() { if (field.Name == "ORDINAL_POSITION") arrays.Add(new Int64Array.Builder().Append(0).Build()); + else if (field.Name == "TABLE_CAT") + arrays.Add(new StringArray.Builder().AppendNull().Build()); else if (field.DataType.TypeId == ArrowTypeId.String) arrays.Add(new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? "DECIMAL(10,2)" : "value").Build()); else if (field.DataType.TypeId == ArrowTypeId.Int8) @@ -55,13 +57,97 @@ public async Task GetColumns_CastsIntegersAndMakesOrdinalOneBased() using var nativeBatch = new RecordBatch(source, arrays.ToArray(), 1); var result = NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, target, MetadataOperation.GetColumns); + new[] { nativeBatch }, target, MetadataOperation.GetColumns, + sourceCatalogs: new[] { "main" }); using var reader = result.Stream!; using var batch = await reader.ReadNextRecordBatchAsync(); Assert.NotNull(batch); - Assert.Equal(1, ((Int32Array)batch.Column(16)).GetValue(0)); + Assert.Equal(10, ((Int32Array)batch.Column(6)).GetValue(0)); + Assert.Equal(2, ((Int32Array)batch.Column(8)).GetValue(0)); + Assert.Equal(0, ((Int32Array)batch.Column(16)).GetValue(0)); + Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); Assert.Equal("DECIMAL", ((StringArray)batch.Column(23)).GetString(0)); } + + [Fact] + public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCatalog() + { + var schema = MetadataSchemaFactory.CreateTablesSchema(); + var arrays = schema.FieldsList.Select(field => (IArrowArray)(field.Name switch + { + "TABLE_CAT" => new StringArray.Builder().AppendNull().Build(), + "TABLE_TYPE" => new StringArray.Builder().Append("").Build(), + _ => new StringArray.Builder().Append(field.Name).Build(), + })).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + var result = NativeMetadataResultBuilder.Build( + new[] { nativeBatch }, schema, MetadataOperation.GetTables, + requestedCatalog: "main", tableTypes: new[] { "TABLE" }); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.Equal(1, result.RowCount); + Assert.Equal("main", ((StringArray)batch!.Column(0)).GetString(0)); + Assert.Equal("TABLE", ((StringArray)batch.Column(3)).GetString(0)); + } + + [Theory] + [InlineData(MetadataOperation.GetCatalogs)] + [InlineData(MetadataOperation.GetSchemas)] + [InlineData(MetadataOperation.GetPrimaryKeys)] + [InlineData(MetadataOperation.GetCrossReference)] + public async Task OtherNativeOperations_PreserveTheirSchemas(MetadataOperation operation) + { + var schema = operation switch + { + MetadataOperation.GetCatalogs => MetadataSchemaFactory.CreateCatalogsSchema(), + MetadataOperation.GetSchemas => MetadataSchemaFactory.CreateSchemasSchema(), + MetadataOperation.GetPrimaryKeys => MetadataSchemaFactory.CreatePrimaryKeysSchema(), + _ => MetadataSchemaFactory.CreateCrossReferenceSchema(), + }; + var arrays = schema.FieldsList.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder().Append("value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(1).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(1).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(1).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(1).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + var result = NativeMetadataResultBuilder.Build(new[] { nativeBatch }, schema, operation); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.Equal(1, result.RowCount); + Assert.Equal(schema.FieldsList.Count, batch!.ColumnCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GetColumns_RejectsWrongColumnNameOrType(bool wrongType) + { + var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); + var fields = target.FieldsList.Take(23).Select(field => + field.Name == "COLUMN_NAME" + ? new Field(wrongType ? field.Name : "wrong_name", wrongType ? (IArrowType)Int32Type.Default : field.DataType, true) + : field).ToArray(); + var schema = new Schema(fields, null); + var arrays = fields.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder().Append("value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + Assert.Throws(() => NativeMetadataResultBuilder.Build( + new[] { nativeBatch }, target, MetadataOperation.GetColumns)); + } } } diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs index 34d571fb..6366791a 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs @@ -266,8 +266,12 @@ public async Task ExecuteStatementAsync_NativeMetadata_SendsHeadersAndReadsManif Assert.Equal("true", Assert.Single(capturedRequest!.Headers.GetValues("x-databricks-sea-can-run-fully-sync"))); Assert.Equal("GetTables", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-metadata-operation-type"))); Assert.Equal("true", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-require-thrift-native-metadata"))); - Assert.DoesNotContain("metadataOperation", capturedContent); - Assert.DoesNotContain("is_native_metadata_result", capturedContent); + using var requestJson = JsonDocument.Parse(capturedContent!); + foreach (var property in requestJson.RootElement.EnumerateObject()) + { + Assert.False(string.Equals(property.Name, "MetadataOperation", StringComparison.OrdinalIgnoreCase)); + Assert.False(string.Equals(property.Name, "IsNativeMetadataResult", StringComparison.OrdinalIgnoreCase)); + } } [Fact] diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 6d15e4ce..9c61f975 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -169,6 +169,46 @@ private static StatementExecutionStatement CreateMetadataStatement( connection: connection); } + [Fact] + public async Task DisableNativeMetadata_LeavesOnlyTheSynchronousMetadataHeader() + { + bool hasSyncHeader = false; + bool hasOperationHeader = false; + bool hasRequireHeader = false; + var handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => + { + if (request.Method == HttpMethod.Post && request.RequestUri?.AbsolutePath.EndsWith("/sql/statements") == true) + { + hasSyncHeader = request.Headers.Contains("x-databricks-sea-can-run-fully-sync"); + hasOperationHeader = request.Headers.Contains("x-databricks-metadata-operation-type"); + hasRequireHeader = request.Headers.Contains("x-databricks-require-thrift-native-metadata"); + } + string body = request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true + ? "{\"session_id\":\"session-1\"}" + : "{\"statement_id\":\"stmt-1\",\"status\":{\"state\":\"SUCCEEDED\"}}"; + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + }); + using var http = new HttpClient(handler.Object); + var properties = new Dictionary + { + [SparkParameters.HostName] = "test.databricks.com", + [DatabricksParameters.WarehouseId] = "wh-1", + [SparkParameters.AccessToken] = "token", + [DatabricksParameters.EnableThriftNativeMetadata] = "false", + }; + using var connection = new StatementExecutionConnection(properties, http); + + await connection.ExecuteNativeMetadataSqlAsync("SHOW CATALOGS", MetadataOperation.GetCatalogs, CancellationToken.None); + + Assert.True(hasSyncHeader); + Assert.False(hasOperationHeader); + Assert.False(hasRequireHeader); + } + // ─── Issue #525: `%` match-all catalog wildcard ────────────────────────────── // The `%` SQL-LIKE wildcard must mean "all catalogs" on the SEA path, exactly as // Thrift treats it, rather than being passed through as a literal backtick-quoted diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs new file mode 100644 index 00000000..c753e575 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -0,0 +1,169 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2.Hive2; +using AdbcDrivers.HiveServer2.Spark; +using Apache.Arrow; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Moq; +using Moq.Protected; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class StatementExecutionNativeCatalogTests + { + private static byte[] ArrowStrings(string name, params string[] values) + { + var schema = new Schema(new[] { new Field(name, StringType.Default, true) }, null); + var builder = new StringArray.Builder(); + foreach (string value in values) builder.Append(value); + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, new IArrowArray[] { builder.Build() }, values.Length)); + writer.WriteEnd(); + } + return raw.ToArray(); + } + + private static HttpClient CreateHttpClient(string? otherFailureState = null) + { + byte[] catalogs = ArrowStrings("TABLE_CAT", "main", "other"); + byte[] columns = ArrowStrings("col_name", "a"); + var handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => + { + if (request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true) + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"session_id\":\"s1\"}") }; + + string sql = ""; + if (request.Method == HttpMethod.Post) + { + using var json = JsonDocument.Parse(request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); + sql = json.RootElement.GetProperty("statement").GetString() ?? ""; + } + if (otherFailureState != null && sql.Contains("`other`")) + { + string failedBody = JsonSerializer.Serialize(new + { + statement_id = "failed", + status = new + { + state = "FAILED", + sql_state = otherFailureState, + error = new { error_code = "QUERY_ERROR", message = "catalog query failed" }, + }, + }); + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent(failedBody), + }; + } + + bool isCatalogs = sql.StartsWith("SHOW CATALOGS"); + string name = isCatalogs ? "TABLE_CAT" : "col_name"; + var body = JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = isCatalogs, + total_row_count = isCatalogs ? 2 : 1, + schema = new + { + column_count = 1, + columns = new[] { new { name, position = 0, type_name = "STRING", type_text = "STRING" } }, + }, + }, + result = new { attachment = isCatalogs ? catalogs : columns }, + }); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + }); + return new HttpClient(handler.Object); + } + + private static StatementExecutionConnection CreateConnection(HttpClient http) + { + var properties = new Dictionary + { + [SparkParameters.HostName] = "test.databricks.com", + [DatabricksParameters.WarehouseId] = "wh-1", + [SparkParameters.AccessToken] = "token", + }; + return new StatementExecutionConnection(properties, http); + } + + [Fact] + public async Task NativeCatalogs_AreFilteredForGetObjects() + { + using var http = CreateHttpClient(); + using var connection = CreateConnection(http); + + var catalogs = await ((IGetObjectsDataProvider)connection).GetCatalogsAsync("main", CancellationToken.None); + + Assert.Equal(new[] { "main" }, catalogs); + } + + [Fact] + public async Task ColumnFanout_KeepsTheQueriedCatalog() + { + using var http = CreateHttpClient(); + using var connection = CreateConnection(http); + + var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); + + Assert.Equal(2, batches.Count); + Assert.Equal("main", batches[0].Catalog); + Assert.Equal("other", batches[1].Catalog); + } + + [Fact] + public async Task ColumnFanout_PropagatesUnrelatedCatalogFailure() + { + using var http = CreateHttpClient(otherFailureState: "XX000"); + using var connection = CreateConnection(http); + + await Assert.ThrowsAsync(() => + connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); + } + + [Fact] + public async Task ColumnFanout_SkipsSqlPermissionDeniedCatalog() + { + using var http = CreateHttpClient(otherFailureState: "42501"); + using var connection = CreateConnection(http); + + var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); + + Assert.Single(batches); + Assert.Equal("main", batches[0].Catalog); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs index 90dd298d..d67532a8 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs @@ -160,6 +160,41 @@ public async Task ExecuteUpdateAsync_FailedAfterPolling_ThrowsDatabricksExceptio Assert.Contains("table not found", exception.Message); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task MetadataResultAfterPolling_KeepsOrDetectsNativeShape(bool initialFlag) + { + _mockClient.Setup(c => c.ExecuteStatementAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ExecuteStatementResponse + { + StatementId = StatementId, + Status = new StatementStatus { State = "PENDING" }, + Manifest = initialFlag ? new ResultManifest { IsNativeMetadataResult = true } : null, + }); + _mockClient.Setup(c => c.GetStatementAsync(StatementId, It.IsAny())) + .ReturnsAsync(new GetStatementResponse + { + StatementId = StatementId, + Status = new StatementStatus { State = "SUCCEEDED" }, + Manifest = new ResultManifest + { + Schema = new ResultSchema + { + ColumnCount = 1, + Columns = new List { new ColumnInfo { Name = "TABLE_CAT", Position = 0, TypeName = "STRING" } }, + }, + }, + }); + + using var stmt = CreateStatement(); + var queryResult = await stmt.ExecuteQueryAsync( + CancellationToken.None, isMetadataExecution: true, metadataOperation: MetadataOperation.GetCatalogs); + using var stream = queryResult.Stream; + + Assert.True(stmt.IsNativeMetadataResult); + } + public void Dispose() { _httpClient?.Dispose(); From 09d95b0fcf168deab9b34ec438d01dee66867d95 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 18:58:02 +0000 Subject: [PATCH 03/22] fix(csharp): compile native metadata across target frameworks Signed-off-by: Vu Anh Phung --- csharp/src/StatementExecution/NativeMetadataColumns.cs | 3 ++- csharp/src/StatementExecution/NativeMetadataResultBuilder.cs | 2 +- csharp/src/StatementExecution/StatementExecutionConnection.cs | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/csharp/src/StatementExecution/NativeMetadataColumns.cs b/csharp/src/StatementExecution/NativeMetadataColumns.cs index 5d3dd4f7..a19eb5f5 100644 --- a/csharp/src/StatementExecution/NativeMetadataColumns.cs +++ b/csharp/src/StatementExecution/NativeMetadataColumns.cs @@ -44,8 +44,9 @@ internal NativeMetadataColumns(RecordBatch batch, Schema schema, MetadataOperati ? array is StringArray : IsInteger(expected.DataType.TypeId) && IsInteger(batch.Schema.FieldsList[index].DataType.TypeId) && array is Int8Array or Int16Array or Int32Array or Int64Array; - if (!valid || !_columns.TryAdd(expected.Name, array)) + if (!valid || _columns.ContainsKey(expected.Name)) throw new DatabricksException($"Invalid native {operation} result: unexpected type or duplicate {expected.Name}"); + _columns.Add(expected.Name, array); } } diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 9b790498..777802d0 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -197,6 +197,6 @@ internal static QueryResult Build( private static bool Matches(string? requested, string? actual) => requested == null || (actual != null && string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase)); - internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value; + internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value!; } } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index a7370628..5c30fa1b 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1074,7 +1074,7 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(batch => (batch, isNative, catalog))); + results.AddRange(batches.Select(batch => (batch, isNative, (string?)catalog))); return results; } @@ -1097,7 +1097,7 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc { var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(columns => (columns, isNative, sourceCatalog))); + results.AddRange(batches.Select(columns => (columns, isNative, (string?)sourceCatalog))); } catch (DatabricksException ex) when (!cancellationToken.IsCancellationRequested && (ex.IsObjectNotFoundException() || IsSqlPermissionDenied(ex))) From 5d821058f29fd07aeb9066affbfa545c65863f06 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 19:01:00 +0000 Subject: [PATCH 04/22] fix(csharp): distinguish catalog misses from native request errors Signed-off-by: Vu Anh Phung --- .../StatementExecutionConnection.cs | 7 ++++++- .../NativeMetadataResultBuilderTests.cs | 11 ++++++----- .../StatementExecutionNativeCatalogTests.cs | 14 ++++++++++++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 5c30fa1b..b264a96e 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1100,7 +1100,7 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc results.AddRange(batches.Select(columns => (columns, isNative, (string?)sourceCatalog))); } catch (DatabricksException ex) when (!cancellationToken.IsCancellationRequested && - (ex.IsObjectNotFoundException() || IsSqlPermissionDenied(ex))) + (IsCatalogObjectNotFound(ex) || IsSqlPermissionDenied(ex))) { System.Diagnostics.Activity.Current?.AddEvent(new System.Diagnostics.ActivityEvent( "statement.get_columns.catalog_skipped", @@ -1119,6 +1119,11 @@ private static bool IsSqlPermissionDenied(DatabricksException ex) => ex.SqlState == "42501" || ex.Message.IndexOf("SQLSTATE: 42501", StringComparison.OrdinalIgnoreCase) >= 0 || ex.Message.IndexOf("INSUFFICIENT_PERMISSIONS", StringComparison.OrdinalIgnoreCase) >= 0; + private static bool IsCatalogObjectNotFound(DatabricksException ex) + => ex.SqlState == "42704" || ex.Message.IndexOf("NO_SUCH_CATALOG_EXCEPTION", StringComparison.OrdinalIgnoreCase) >= 0 + || ex.Message.IndexOf("SCHEMA_NOT_FOUND", StringComparison.OrdinalIgnoreCase) >= 0 + || ex.Message.IndexOf("TABLE_OR_VIEW_NOT_FOUND", StringComparison.OrdinalIgnoreCase) >= 0; + private static bool MatchesCatalogPattern(string? pattern, string catalog) { if (pattern == null) return true; diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index e7465758..056a0f1d 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -94,12 +94,13 @@ public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCat } [Theory] - [InlineData(MetadataOperation.GetCatalogs)] - [InlineData(MetadataOperation.GetSchemas)] - [InlineData(MetadataOperation.GetPrimaryKeys)] - [InlineData(MetadataOperation.GetCrossReference)] - public async Task OtherNativeOperations_PreserveTheirSchemas(MetadataOperation operation) + [InlineData((int)MetadataOperation.GetCatalogs)] + [InlineData((int)MetadataOperation.GetSchemas)] + [InlineData((int)MetadataOperation.GetPrimaryKeys)] + [InlineData((int)MetadataOperation.GetCrossReference)] + public async Task OtherNativeOperations_PreserveTheirSchemas(int operationCode) { + var operation = (MetadataOperation)operationCode; var schema = operation switch { MetadataOperation.GetCatalogs => MetadataSchemaFactory.CreateCatalogsSchema(), diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index c753e575..4383d0e8 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -49,7 +49,7 @@ private static byte[] ArrowStrings(string name, params string[] values) return raw.ToArray(); } - private static HttpClient CreateHttpClient(string? otherFailureState = null) + private static HttpClient CreateHttpClient(string? otherFailureState = null, string otherFailureCode = "QUERY_ERROR") { byte[] catalogs = ArrowStrings("TABLE_CAT", "main", "other"); byte[] columns = ArrowStrings("col_name", "a"); @@ -77,7 +77,7 @@ private static HttpClient CreateHttpClient(string? otherFailureState = null) { state = "FAILED", sql_state = otherFailureState, - error = new { error_code = "QUERY_ERROR", message = "catalog query failed" }, + error = new { error_code = otherFailureCode, message = "catalog query failed" }, }, }); return new HttpResponseMessage(HttpStatusCode.OK) @@ -165,5 +165,15 @@ public async Task ColumnFanout_SkipsSqlPermissionDeniedCatalog() Assert.Single(batches); Assert.Equal("main", batches[0].Catalog); } + + [Fact] + public async Task ColumnFanout_PropagatesNativeRequirementRejection() + { + using var http = CreateHttpClient(otherFailureState: "22023", otherFailureCode: "INVALID_PARAMETER_VALUE"); + using var connection = CreateConnection(http); + + await Assert.ThrowsAsync(() => + connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); + } } } From b4d40ea0fcd2f1fc1acf89a92c62d6fac28b2b07 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 19:04:53 +0000 Subject: [PATCH 05/22] test(csharp): cover native GetObjects catalog filtering Signed-off-by: Vu Anh Phung --- .../StatementExecutionNativeCatalogTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 4383d0e8..3e6ca25a 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -25,6 +25,7 @@ using AdbcDrivers.HiveServer2.Hive2; using AdbcDrivers.HiveServer2.Spark; using Apache.Arrow; +using Apache.Arrow.Adbc; using Apache.Arrow.Ipc; using Apache.Arrow.Types; using Moq; @@ -131,6 +132,20 @@ public async Task NativeCatalogs_AreFilteredForGetObjects() Assert.Equal(new[] { "main" }, catalogs); } + [Fact] + public async Task GetObjects_CatalogDepth_FiltersNativeCatalogs() + { + using var http = CreateHttpClient(); + using var connection = CreateConnection(http); + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.Catalogs, "main", null, null, null, null); + using var batch = await stream.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + Assert.Equal(1, batch.Length); + Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); + } + [Fact] public async Task ColumnFanout_KeepsTheQueriedCatalog() { From 464686d6bd2b81d228eaf6851553376653f52007 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 19:17:06 +0000 Subject: [PATCH 06/22] refactor(csharp): always request native SEA metadata Signed-off-by: Vu Anh Phung --- csharp/README.md | 1 - csharp/src/DatabricksParameters.cs | 7 ------- .../src/StatementExecution/StatementExecutionConnection.cs | 4 ---- .../src/StatementExecution/StatementExecutionStatement.cs | 2 +- .../StatementExecutionMetadataObjectNotFoundTests.cs | 7 +++---- 5 files changed, 4 insertions(+), 17 deletions(-) diff --git a/csharp/README.md b/csharp/README.md index b424d166..315866ce 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -218,7 +218,6 @@ The Databricks driver supports two protocols: **Thrift** (default, HiveServer2) | `adbc.databricks.max_bytes_per_fetch_request` | Max bytes per fetch request (supports B, KB, MB, GB) | `400MB` | Not Supported | | `adbc.databricks.apply_ssp_with_queries` | Apply server-side properties with queries | `false` | Not Supported | | `adbc.databricks.enable_multiple_catalog_support` | Support multiple catalogs | `true` | `true` | -| `adbc.databricks.enable_thrift_native_metadata` | Require native metadata results for SEA metadata queries; set `false` to use SHOW results | Not Supported | `true` | | `adbc.databricks.enable_pk_fk` | Enable primary/foreign key metadata | `true` | `true` | | `adbc.databricks.use_desc_table_extended` | Use DESC TABLE EXTENDED when supported | `true` | `true` | | `adbc.databricks.enable_run_async_thrift` | Enable RunAsync flag | `true` | Not Supported (Thrift-only concept) | diff --git a/csharp/src/DatabricksParameters.cs b/csharp/src/DatabricksParameters.cs index 76aad46a..8e56b2db 100644 --- a/csharp/src/DatabricksParameters.cs +++ b/csharp/src/DatabricksParameters.cs @@ -339,13 +339,6 @@ public class DatabricksParameters : SparkParameters [FeatureFlagType(FeatureFlagValueKind.Boolean)] public const string EnableFastMetadataQuery = "adbc.databricks.enable_fast_metadata_query"; - /// - /// Require Thrift-shaped native metadata results on Statement Execution metadata queries. - /// Default value is true. Set false to use SHOW results instead. - /// - [FeatureFlagType(FeatureFlagValueKind.Boolean)] - public const string EnableThriftNativeMetadata = "adbc.databricks.enable_thrift_native_metadata"; - /// /// Whether to enable RunAsync flag in Thrift operation /// Default value is true if not specified. diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index b264a96e..b5ca2283 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -72,7 +72,6 @@ internal class StatementExecutionConnection : TracingConnection, IGetObjectsData private bool _scopeCurrentCatalog; private bool _useDescTableExtended; private bool _enableFastMetadataQuery; - private bool _enableThriftNativeMetadata; private bool _applySSPWithQueries; // Connection bring-up timeout (PECO-3059). Mirrors the Thrift path's @@ -329,7 +328,6 @@ private void ValidateProperties() _scopeCurrentCatalog = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.ScopeCurrentCatalog, false); _useDescTableExtended = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.UseDescTableExtended, true); _enableFastMetadataQuery = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.EnableFastMetadataQuery, false); - _enableThriftNativeMetadata = PropertyHelper.GetBooleanPropertyWithValidation(properties, DatabricksParameters.EnableThriftNativeMetadata, true); // When true, SSPs (adbc.databricks.ssp_*) are applied via post-open SET statements // rather than CreateSession.session_confs — mirrors Thrift's behavior so callers // who depend on the SET-statement path (e.g., for audit visibility or for SSPs @@ -1194,8 +1192,6 @@ private static bool MatchesCatalogPattern(string? pattern, string catalog) /// internal bool EnableFastMetadataQuery => _enableFastMetadataQuery; - internal bool EnableThriftNativeMetadata => _enableThriftNativeMetadata; - /// /// Returns the session's default catalog by querying the server via /// SELECT CURRENT_CATALOG(). SEA's CreateSession response, unlike Thrift's diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 881e1580..50ef37bc 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -417,7 +417,7 @@ private async Task ExecuteQueryInternalAsync( WaitTimeout = _waitTimeout, OnWaitTimeout = "CONTINUE", IsMetadata = isMetadataExecution, - MetadataOperation = _connection.EnableThriftNativeMetadata ? metadataOperation : null, + MetadataOperation = metadataOperation, QueryTags = ParseQueryTags(_queryTags) }; diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 9c61f975..7a3972e7 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -170,7 +170,7 @@ private static StatementExecutionStatement CreateMetadataStatement( } [Fact] - public async Task DisableNativeMetadata_LeavesOnlyTheSynchronousMetadataHeader() + public async Task MetadataConnection_AlwaysRequestsNativeMetadata() { bool hasSyncHeader = false; bool hasOperationHeader = false; @@ -198,15 +198,14 @@ public async Task DisableNativeMetadata_LeavesOnlyTheSynchronousMetadataHeader() [SparkParameters.HostName] = "test.databricks.com", [DatabricksParameters.WarehouseId] = "wh-1", [SparkParameters.AccessToken] = "token", - [DatabricksParameters.EnableThriftNativeMetadata] = "false", }; using var connection = new StatementExecutionConnection(properties, http); await connection.ExecuteNativeMetadataSqlAsync("SHOW CATALOGS", MetadataOperation.GetCatalogs, CancellationToken.None); Assert.True(hasSyncHeader); - Assert.False(hasOperationHeader); - Assert.False(hasRequireHeader); + Assert.True(hasOperationHeader); + Assert.True(hasRequireHeader); } // ─── Issue #525: `%` match-all catalog wildcard ────────────────────────────── From 0ca3b38eeeb61d40f6cd9d1511d1d49303c73fd9 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 21:33:54 +0000 Subject: [PATCH 07/22] fix(csharp): honor literal catalog filters in native metadata Signed-off-by: Vu Anh Phung --- .../NativeMetadataResultBuilder.cs | 14 +++++-- .../StatementExecutionStatement.cs | 7 +++- .../NativeMetadataResultBuilderTests.cs | 39 +++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 777802d0..b8a431a9 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -31,12 +31,18 @@ internal static QueryResult Build( IReadOnlyList batches, Schema schema, MetadataOperation operation, string? requestedCatalog = null, IReadOnlyCollection? tableTypes = null, string? parentCatalog = null, string? parentSchema = null, string? parentTable = null, - IReadOnlyList? sourceCatalogs = null) + IReadOnlyList? sourceCatalogs = null, bool requireExactCatalog = false) { // C# exposes BASE_TYPE_NAME after the 23 Thrift GetColumns fields. int sourceColumns = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); if (sourceCatalogs != null && sourceCatalogs.Count != batches.Count) throw new ArgumentException("Each native batch must have a source catalog", nameof(sourceCatalogs)); + string? catalogField = operation switch + { + MetadataOperation.GetSchemas => "TABLE_CATALOG", + MetadataOperation.GetTables or MetadataOperation.GetColumns => "TABLE_CAT", + _ => null, + }; var rows = new List<(NativeMetadataColumns Columns, int Index, string? Catalog)>(); for (int batchIndex = 0; batchIndex < batches.Count; batchIndex++) { @@ -46,8 +52,10 @@ internal static QueryResult Build( for (int row = 0; row < batch.Length; row++) { - if (operation == MetadataOperation.GetTables && requestedCatalog != null && - !string.Equals(requestedCatalog, columns.String("TABLE_CAT", row) ?? sourceCatalog, StringComparison.OrdinalIgnoreCase)) + // Native metadata can expand wildcard characters in a quoted catalog. + if (catalogField != null && requestedCatalog != null && + (operation == MetadataOperation.GetTables || requireExactCatalog) && + !string.Equals(requestedCatalog, columns.String(catalogField, row) ?? sourceCatalog, StringComparison.OrdinalIgnoreCase)) continue; if (operation == MetadataOperation.GetTables && tableTypes != null && !tableTypes.Contains(DefaultTableType(columns.String("TABLE_TYPE", row)))) diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 50ef37bc..1fa0e968 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1378,7 +1378,7 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo if (nativeResult.IsNative) return NativeMetadataResultBuilder.Build( batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, - requestedCatalog: catalog); + requestedCatalog: catalog, requireExactCatalog: _escapePatternWildcards); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -1588,7 +1588,8 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo columnBatches.Select(result => result.Batch).ToList(), MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, requestedCatalog: catalog, - sourceCatalogs: columnBatches.Select(result => result.Catalog).ToList()); + sourceCatalogs: columnBatches.Select(result => result.Catalog).ToList(), + requireExactCatalog: _escapePatternWildcards); } var tableInfos = new Dictionary(); @@ -1619,6 +1620,8 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo string? colType = isNative ? native!.String("TYPE_NAME", i) : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; + if (isNative && _escapePatternWildcards && catalog != null && + !string.Equals(catalog, cat, StringComparison.OrdinalIgnoreCase)) continue; string key = $"{cat}.{sch}.{tbl}"; diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index 056a0f1d..2ce5c130 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -93,6 +93,45 @@ public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCat Assert.Equal("TABLE", ((StringArray)batch.Column(3)).GetString(0)); } + [Theory] + [InlineData((int)MetadataOperation.GetSchemas, "%", true, 0)] + [InlineData((int)MetadataOperation.GetColumns, "compar%", true, 0)] + [InlineData((int)MetadataOperation.GetColumns, @"comparator\_tests", true, 0)] + [InlineData((int)MetadataOperation.GetSchemas, "comparator_tests", true, 1)] + [InlineData((int)MetadataOperation.GetColumns, "COMPARATOR_TESTS", true, 1)] + [InlineData((int)MetadataOperation.GetSchemas, "compar%", false, 2)] + public void NativeSchemasAndColumns_FilterLiteralCatalogsOnlyWhenRequested( + int operationCode, string requestedCatalog, bool requireExactCatalog, int expectedRows) + { + var operation = (MetadataOperation)operationCode; + var target = operation == MetadataOperation.GetSchemas + ? MetadataSchemaFactory.CreateSchemasSchema() + : MetadataSchemaFactory.CreateColumnMetadataSchema(); + var fields = operation == MetadataOperation.GetColumns + ? target.FieldsList.Take(23).ToArray() + : target.FieldsList.ToArray(); + string catalogField = operation == MetadataOperation.GetSchemas ? "TABLE_CATALOG" : "TABLE_CAT"; + var source = new Schema(fields, null); + var arrays = fields.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String when field.Name == catalogField => + (IArrowArray)new StringArray.Builder().Append("comparator_tests").Append("comparator-tests").Build(), + ArrowTypeId.String => new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? "INT" : "value") + .Append(field.Name == "TYPE_NAME" ? "INT" : "value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Append(0).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(source, arrays, 2); + + var result = NativeMetadataResultBuilder.Build( + new[] { nativeBatch }, target, operation, + requestedCatalog: requestedCatalog, requireExactCatalog: requireExactCatalog); + + Assert.Equal(expectedRows, result.RowCount); + } + [Theory] [InlineData((int)MetadataOperation.GetCatalogs)] [InlineData((int)MetadataOperation.GetSchemas)] From 8c8ad67d078c1e468ab886571dc3c435ac98d16a Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 22:41:24 +0000 Subject: [PATCH 08/22] refactor(csharp): trust manifest for native metadata detection Signed-off-by: Vu Anh Phung --- .../StatementExecutionStatement.cs | 28 +------------------ ...atementExecutionPollingFailedStateTests.cs | 17 +++++++---- 2 files changed, 13 insertions(+), 32 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 1fa0e968..c7cf9c40 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -473,7 +473,7 @@ private async Task ExecuteQueryInternalAsync( IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult ?? initialNativeMetadata - ?? HasNativeMetadataSchema(response.Manifest?.Schema, metadataOperation); + ?? false; // Check for truncated results warning if (response.Manifest?.Truncated == true) @@ -517,32 +517,6 @@ private async Task ExecuteQueryInternalAsync( return new QueryResult(rowCount, reader); } - private static bool HasNativeMetadataSchema(ResultSchema? resultSchema, MetadataOperation? operation) - { - if (operation == null || resultSchema?.Columns == null) - return false; - - Schema expected = operation.Value switch - { - MetadataOperation.GetCatalogs => MetadataSchemaFactory.CreateCatalogsSchema(), - MetadataOperation.GetSchemas => MetadataSchemaFactory.CreateSchemasSchema(), - MetadataOperation.GetTables => MetadataSchemaFactory.CreateTablesSchema(), - MetadataOperation.GetColumns => MetadataSchemaFactory.CreateColumnMetadataSchema(), - MetadataOperation.GetPrimaryKeys => MetadataSchemaFactory.CreatePrimaryKeysSchema(), - MetadataOperation.GetCrossReference => MetadataSchemaFactory.CreateCrossReferenceSchema(), - _ => throw new ArgumentOutOfRangeException(nameof(operation)), - }; - int count = expected.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); - if (resultSchema.Columns.Count != count) - return false; - for (int i = 0; i < count; i++) - { - if (!string.Equals(resultSchema.Columns[i].Name, expected.FieldsList[i].Name, StringComparison.Ordinal)) - return false; - } - return true; - } - /// /// Wraps PollUntilCompleteAsync with query timeout enforcement. /// If _queryTimeoutSeconds > 0, cancels the server-side statement and throws on timeout. diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs index d67532a8..4181eaf3 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs @@ -161,16 +161,22 @@ public async Task ExecuteUpdateAsync_FailedAfterPolling_ThrowsDatabricksExceptio } [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task MetadataResultAfterPolling_KeepsOrDetectsNativeShape(bool initialFlag) + [InlineData(true, null, true)] + [InlineData(null, true, true)] + [InlineData(true, false, false)] + [InlineData(null, false, false)] + [InlineData(null, null, false)] + public async Task MetadataResultAfterPolling_UsesManifestMarker( + bool? initialFlag, bool? finalFlag, bool expectedNative) { _mockClient.Setup(c => c.ExecuteStatementAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(new ExecuteStatementResponse { StatementId = StatementId, Status = new StatementStatus { State = "PENDING" }, - Manifest = initialFlag ? new ResultManifest { IsNativeMetadataResult = true } : null, + Manifest = initialFlag.HasValue + ? new ResultManifest { IsNativeMetadataResult = initialFlag } + : null, }); _mockClient.Setup(c => c.GetStatementAsync(StatementId, It.IsAny())) .ReturnsAsync(new GetStatementResponse @@ -179,6 +185,7 @@ public async Task MetadataResultAfterPolling_KeepsOrDetectsNativeShape(bool init Status = new StatementStatus { State = "SUCCEEDED" }, Manifest = new ResultManifest { + IsNativeMetadataResult = finalFlag, Schema = new ResultSchema { ColumnCount = 1, @@ -192,7 +199,7 @@ public async Task MetadataResultAfterPolling_KeepsOrDetectsNativeShape(bool init CancellationToken.None, isMetadataExecution: true, metadataOperation: MetadataOperation.GetCatalogs); using var stream = queryResult.Stream; - Assert.True(stmt.IsNativeMetadataResult); + Assert.Equal(expectedNative, stmt.IsNativeMetadataResult); } public void Dispose() From c22a1d1eb2dbb576a6901d12c5c957ffb0a8e764 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 22:50:48 +0000 Subject: [PATCH 09/22] Restore existing catalog fan-out error behavior Signed-off-by: Vu Anh Phung --- .../StatementExecutionConnection.cs | 20 +------ .../StatementExecutionNativeCatalogTests.cs | 52 +------------------ 2 files changed, 3 insertions(+), 69 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index b5ca2283..3edf90c3 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1097,31 +1097,15 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); results.AddRange(batches.Select(columns => (columns, isNative, (string?)sourceCatalog))); } - catch (DatabricksException ex) when (!cancellationToken.IsCancellationRequested && - (IsCatalogObjectNotFound(ex) || IsSqlPermissionDenied(ex))) + catch { - System.Diagnostics.Activity.Current?.AddEvent(new System.Diagnostics.ActivityEvent( - "statement.get_columns.catalog_skipped", - tags: new System.Diagnostics.ActivityTagsCollection - { - { "catalog", sourceCatalog }, - { "error", ex.Message }, - })); + // Skip catalogs we can't access (permission errors) } } } return results; } - private static bool IsSqlPermissionDenied(DatabricksException ex) - => ex.SqlState == "42501" || ex.Message.IndexOf("SQLSTATE: 42501", StringComparison.OrdinalIgnoreCase) >= 0 - || ex.Message.IndexOf("INSUFFICIENT_PERMISSIONS", StringComparison.OrdinalIgnoreCase) >= 0; - - private static bool IsCatalogObjectNotFound(DatabricksException ex) - => ex.SqlState == "42704" || ex.Message.IndexOf("NO_SUCH_CATALOG_EXCEPTION", StringComparison.OrdinalIgnoreCase) >= 0 - || ex.Message.IndexOf("SCHEMA_NOT_FOUND", StringComparison.OrdinalIgnoreCase) >= 0 - || ex.Message.IndexOf("TABLE_OR_VIEW_NOT_FOUND", StringComparison.OrdinalIgnoreCase) >= 0; - private static bool MatchesCatalogPattern(string? pattern, string catalog) { if (pattern == null) return true; diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 3e6ca25a..9f300342 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -50,7 +50,7 @@ private static byte[] ArrowStrings(string name, params string[] values) return raw.ToArray(); } - private static HttpClient CreateHttpClient(string? otherFailureState = null, string otherFailureCode = "QUERY_ERROR") + private static HttpClient CreateHttpClient() { byte[] catalogs = ArrowStrings("TABLE_CAT", "main", "other"); byte[] columns = ArrowStrings("col_name", "a"); @@ -69,24 +69,6 @@ private static HttpClient CreateHttpClient(string? otherFailureState = null, str using var json = JsonDocument.Parse(request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); sql = json.RootElement.GetProperty("statement").GetString() ?? ""; } - if (otherFailureState != null && sql.Contains("`other`")) - { - string failedBody = JsonSerializer.Serialize(new - { - statement_id = "failed", - status = new - { - state = "FAILED", - sql_state = otherFailureState, - error = new { error_code = otherFailureCode, message = "catalog query failed" }, - }, - }); - return new HttpResponseMessage(HttpStatusCode.OK) - { - Content = new StringContent(failedBody), - }; - } - bool isCatalogs = sql.StartsWith("SHOW CATALOGS"); string name = isCatalogs ? "TABLE_CAT" : "col_name"; var body = JsonSerializer.Serialize(new @@ -158,37 +140,5 @@ public async Task ColumnFanout_KeepsTheQueriedCatalog() Assert.Equal("main", batches[0].Catalog); Assert.Equal("other", batches[1].Catalog); } - - [Fact] - public async Task ColumnFanout_PropagatesUnrelatedCatalogFailure() - { - using var http = CreateHttpClient(otherFailureState: "XX000"); - using var connection = CreateConnection(http); - - await Assert.ThrowsAsync(() => - connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); - } - - [Fact] - public async Task ColumnFanout_SkipsSqlPermissionDeniedCatalog() - { - using var http = CreateHttpClient(otherFailureState: "42501"); - using var connection = CreateConnection(http); - - var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); - - Assert.Single(batches); - Assert.Equal("main", batches[0].Catalog); - } - - [Fact] - public async Task ColumnFanout_PropagatesNativeRequirementRejection() - { - using var http = CreateHttpClient(otherFailureState: "22023", otherFailureCode: "INVALID_PARAMETER_VALUE"); - using var connection = CreateConnection(http); - - await Assert.ThrowsAsync(() => - connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); - } } } From 5194948f5ab2d82c95745c3cc578d33dd5d42c6a Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 23:07:44 +0000 Subject: [PATCH 10/22] Filter native metadata column fan-out by catalog Signed-off-by: Vu Anh Phung --- .../StatementExecutionConnection.cs | 28 ++++- .../StatementExecutionStatement.cs | 2 +- .../StatementExecutionNativeCatalogTests.cs | 113 ++++++++++++++++-- 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 3edf90c3..28421c2a 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1008,7 +1008,7 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter ColumnMetadataHelper.PopulateTableInfoFromTypeName( tableInfo, colName, colType, position, nullable, columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, - isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTOINCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); + isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTO_INCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); } } } @@ -1095,7 +1095,31 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc { var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(columns => (columns, isNative, (string?)sourceCatalog))); + foreach (var columns in batches) + { + if (!isNative) + { + results.Add((columns, false, sourceCatalog)); + continue; + } + + // Native GetColumns can treat catalog names as LIKE patterns. + var native = new NativeMetadataColumns( + columns, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); + int start = -1; + for (int row = 0; row <= columns.Length; row++) + { + string? rowCatalog = row < columns.Length ? native.String("TABLE_CAT", row) : null; + bool matches = row < columns.Length && (rowCatalog == null || + string.Equals(rowCatalog, sourceCatalog, StringComparison.OrdinalIgnoreCase)); + if (matches && start < 0) start = row; + if (!matches && start >= 0) + { + results.Add((columns.Slice(start, row - start), true, sourceCatalog)); + start = -1; + } + } + } } catch { diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index c7cf9c40..5ec966a1 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1618,7 +1618,7 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo position, nullable, columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, - isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTOINCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); + isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTO_INCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); } } diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 9f300342..e8821319 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -16,12 +16,14 @@ using System.Collections.Generic; using System.IO; +using System.Linq; using System.Net; using System.Net.Http; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; using AdbcDrivers.HiveServer2.Hive2; using AdbcDrivers.HiveServer2.Spark; using Apache.Arrow; @@ -50,10 +52,50 @@ private static byte[] ArrowStrings(string name, params string[] values) return raw.ToArray(); } - private static HttpClient CreateHttpClient() + private static byte[] ArrowNativeColumns(Field[] fields) { - byte[] catalogs = ArrowStrings("TABLE_CAT", "main", "other"); - byte[] columns = ArrowStrings("col_name", "a"); + var schema = new Schema(fields, null); + var arrays = fields.Select(field => + { + if (field.Name == "TABLE_CAT") + return (IArrowArray)new StringArray.Builder().Append("foo_bar").Append("fooxbar").Build(); + string value = field.Name switch + { + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "COLUMN_NAME" => "a", + "TYPE_NAME" => "INT", + "IS_AUTO_INCREMENT" => "YES", + _ => field.Name, + }; + return field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder() + .Append(value).Append(value).Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Append(0).Build(), + }; + }).ToArray(); + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, arrays, 2)); + writer.WriteEnd(); + } + return raw.ToArray(); + } + + private static HttpClient CreateHttpClient(bool overlappingNativeColumns = false) + { + byte[] catalogs = overlappingNativeColumns + ? ArrowStrings("TABLE_CAT", "foo_bar", "fooxbar") + : ArrowStrings("TABLE_CAT", "main", "other"); + Field[] nativeColumnFields = MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); + byte[] columns = overlappingNativeColumns + ? ArrowNativeColumns(nativeColumnFields) + : ArrowStrings("col_name", "a"); var handler = new Mock(); handler.Protected() .Setup>("SendAsync", @@ -70,19 +112,35 @@ private static HttpClient CreateHttpClient() sql = json.RootElement.GetProperty("statement").GetString() ?? ""; } bool isCatalogs = sql.StartsWith("SHOW CATALOGS"); - string name = isCatalogs ? "TABLE_CAT" : "col_name"; + Field[] fields = isCatalogs + ? new[] { new Field("TABLE_CAT", StringType.Default, true) } + : overlappingNativeColumns + ? nativeColumnFields + : new[] { new Field("col_name", StringType.Default, true) }; + var manifestColumns = fields.Select((field, position) => + { + string type = field.DataType.TypeId switch + { + ArrowTypeId.Int8 => "TINYINT", + ArrowTypeId.Int16 => "SMALLINT", + ArrowTypeId.Int32 => "INT", + ArrowTypeId.Int64 => "BIGINT", + _ => "STRING", + }; + return new { name = field.Name, position, type_name = type, type_text = type }; + }).ToArray(); var body = JsonSerializer.Serialize(new { statement_id = "stmt-1", status = new { state = "SUCCEEDED" }, manifest = new { - is_native_metadata_result = isCatalogs, - total_row_count = isCatalogs ? 2 : 1, + is_native_metadata_result = isCatalogs || overlappingNativeColumns, + total_row_count = isCatalogs || overlappingNativeColumns ? 2 : 1, schema = new { - column_count = 1, - columns = new[] { new { name, position = 0, type_name = "STRING", type_text = "STRING" } }, + column_count = fields.Length, + columns = manifestColumns, }, }, result = new { attachment = isCatalogs ? catalogs : columns }, @@ -140,5 +198,44 @@ public async Task ColumnFanout_KeepsTheQueriedCatalog() Assert.Equal("main", batches[0].Catalog); Assert.Equal("other", batches[1].Catalog); } + + [Fact] + public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() + { + using var http = CreateHttpClient(overlappingNativeColumns: true); + using var connection = CreateConnection(http); + + var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); + + Assert.Equal(2, batches.Count); + Assert.All(batches, result => + { + Assert.True(result.IsNative); + Assert.Equal(1, result.Batch.Length); + Assert.Equal(result.Catalog, ((StringArray)result.Batch.Column("TABLE_CAT")).GetString(0)); + }); + Assert.Equal(new[] { "foo_bar", "fooxbar" }, batches.Select(result => result.Catalog)); + + var result = NativeMetadataResultBuilder.Build( + batches.Select(batch => batch.Batch).ToArray(), + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, + sourceCatalogs: batches.Select(batch => batch.Catalog).ToArray()); + using var stream = result.Stream!; + Assert.Equal(2, result.RowCount); + + var catalogMap = new Dictionary>>(); + foreach (string catalog in new[] { "foo_bar", "fooxbar" }) + { + catalogMap[catalog] = new Dictionary> + { + ["default"] = new Dictionary { ["t"] = new TableInfo("TABLE") }, + }; + } + await ((IGetObjectsDataProvider)connection).PopulateColumnInfoAsync( + null, null, null, null, catalogMap, CancellationToken.None); + Assert.Single(catalogMap["foo_bar"]["default"]["t"].ColumnName); + Assert.Single(catalogMap["fooxbar"]["default"]["t"].ColumnName); + Assert.True(catalogMap["foo_bar"]["default"]["t"].IsAutoIncrement.Single()); + } } } From 1af4a1a2264b9cf3f1127a7896960ddcd320c23b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Thu, 1 Oct 2026 23:10:17 +0000 Subject: [PATCH 11/22] Fix native column filter local name collision Signed-off-by: Vu Anh Phung --- csharp/src/StatementExecution/StatementExecutionConnection.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 28421c2a..4dda9363 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1104,12 +1104,12 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc } // Native GetColumns can treat catalog names as LIKE patterns. - var native = new NativeMetadataColumns( + var nativeColumns = new NativeMetadataColumns( columns, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); int start = -1; for (int row = 0; row <= columns.Length; row++) { - string? rowCatalog = row < columns.Length ? native.String("TABLE_CAT", row) : null; + string? rowCatalog = row < columns.Length ? nativeColumns.String("TABLE_CAT", row) : null; bool matches = row < columns.Length && (rowCatalog == null || string.Equals(rowCatalog, sourceCatalog, StringComparison.OrdinalIgnoreCase)); if (matches && start < 0) start = row; From 967a1936c9cc8bbdcc287e28c169e44542f246e7 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Tue, 6 Oct 2026 18:59:19 +0000 Subject: [PATCH 12/22] fix(csharp): resolve GetObjects catalog patterns and validate native metadata Signed-off-by: Vu Anh Phung --- csharp/README.md | 4 + .../StatementExecutionConnection.cs | 123 +++++--- .../StatementExecutionStatement.cs | 12 +- .../test/E2E/DatabricksTestConfiguration.cs | 3 + .../StatementExecution/SeaMetadataE2ETests.cs | 40 ++- ...tExecutionGetObjectsCatalogPatternTests.cs | 281 ++++++++++++++++++ 6 files changed, 423 insertions(+), 40 deletions(-) create mode 100644 csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs diff --git a/csharp/README.md b/csharp/README.md index 315866ce..664b660e 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -612,6 +612,10 @@ Run tests matching a pattern: dotnet test --filter "FullyQualifiedName~CloudFetch" ``` +For a native-capable warehouse, set `"requireNativeMetadata": true` in the test +configuration and run `dotnet test --filter "FullyQualifiedName~NativeMetadata_UsesNativeResponse"`. +These tests fail if any supported metadata operation falls back to SHOW. + #### Verbose Test Output Get detailed test output: diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 4dda9363..8b3d8d54 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -767,6 +767,9 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string async Task> IGetObjectsDataProvider.GetCatalogsAsync(string? catalogPattern, CancellationToken cancellationToken) { + if (catalogPattern == "") + return System.Array.Empty(); + string sql = new ShowCatalogsCommand(catalogPattern).Build(); List batches; bool isNative; @@ -791,7 +794,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin { string? name = isNative ? native!.String("TABLE_CAT", i) : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - if (name != null && (!isNative || MatchesCatalogPattern(catalogPattern, name))) + if (name != null && MatchesCatalogPattern(catalogPattern, name)) result.Add(name); } } @@ -800,11 +803,18 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) { - // Note: catalogPattern comes from GetObjectsResultBuilder which resolves individual - // catalog names before calling this method. Despite the "pattern" name (from the - // IGetObjectsDataProvider interface), the value passed to ShowSchemasCommand is used - // as a literal catalog identifier (backtick-quoted), not a wildcard pattern. - string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); + var result = new List<(string catalog, string schema)>(); + foreach (string? catalog in await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false)) + { + result.AddRange(await GetSchemasInCatalogAsync(catalog, schemaPattern, cancellationToken).ConfigureAwait(false)); + } + return result; + } + + private async Task> GetSchemasInCatalogAsync( + string? catalog, string? schemaPattern, CancellationToken cancellationToken) + { + string sql = new ShowSchemasCommand(catalog, schemaPattern).Build(); List batches; bool isNative; @@ -820,7 +830,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showSchemasInAllCatalogs = catalogPattern == null; + bool showSchemasInAllCatalogs = catalog == null; var result = new List<(string, string)>(); foreach (var batch in batches) @@ -834,8 +844,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin for (int i = 0; i < batch.Length; i++) { string? name = native.String("TABLE_SCHEM", i); - if (name != null) - result.Add((native.String("TABLE_CATALOG", i) ?? catalogPattern ?? "", name)); + string? rowCatalog = native.String("TABLE_CATALOG", i) ?? catalog; + if (name != null && (catalog == null || + string.Equals(catalog, rowCatalog, StringComparison.OrdinalIgnoreCase))) + result.Add((rowCatalog ?? "", name)); } continue; } @@ -853,10 +865,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin for (int i = 0; i < batch.Length; i++) { if (schemaArray.IsNull(i)) continue; - string catalog = catalogArray != null && !catalogArray.IsNull(i) + string rowCatalog = catalogArray != null && !catalogArray.IsNull(i) ? catalogArray.GetString(i) - : catalogPattern ?? ""; - result.Add((catalog, schemaArray.GetString(i))); + : catalog ?? ""; + result.Add((rowCatalog, schemaArray.GetString(i))); } } return result; @@ -875,7 +887,23 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin return System.Array.Empty<(string, string, string, string)>(); } - string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); + IReadOnlyList catalogs = await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false); + var result = new List<(string catalog, string schema, string table, string tableType)>(); + foreach (string? catalog in catalogs) + { + result.AddRange(await GetTablesInCatalogAsync( + catalog, schemaPattern, tableNamePattern, tableTypes, cancellationToken).ConfigureAwait(false)); + } + if (catalogs.Count > 1) + result.Sort(CompareTables); + return result; + } + + private async Task> GetTablesInCatalogAsync( + string? catalog, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes, + CancellationToken cancellationToken) + { + string sql = new ShowTablesCommand(catalog, schemaPattern, tableNamePattern).Build(); List batches; bool isNative; @@ -901,7 +929,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin for (int i = 0; i < batch.Length; i++) { - string? cat = isNative ? native!.String("TABLE_CAT", i) ?? catalogPattern + string? cat = isNative ? native!.String("TABLE_CAT", i) ?? catalog : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); string? sch = isNative ? native!.String("TABLE_SCHEM", i) : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); @@ -909,8 +937,8 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin : tableArray!.IsNull(i) ? null : tableArray.GetString(i); if ((!isNative && cat == null) || sch == null || tbl == null) continue; cat ??= ""; - if (isNative && catalogPattern != null && - !string.Equals(catalogPattern, cat, StringComparison.OrdinalIgnoreCase)) continue; + if (isNative && catalog != null && + !string.Equals(catalog, cat, StringComparison.OrdinalIgnoreCase)) continue; string tableType = NativeMetadataResultBuilder.DefaultTableType(isNative ? native!.String("TABLE_TYPE", i) @@ -926,36 +954,39 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin } } if (isNative) - { - result.Sort((left, right) => - { - int order = StringComparer.Ordinal.Compare(left.tableType, right.tableType); - if (order != 0) return order; - order = StringComparer.Ordinal.Compare(left.catalog, right.catalog); - if (order != 0) return order; - order = StringComparer.Ordinal.Compare(left.schema, right.schema); - return order != 0 ? order : StringComparer.Ordinal.Compare(left.table, right.table); - }); - } + result.Sort(CompareTables); return result; } + private static int CompareTables( + (string catalog, string schema, string table, string tableType) left, + (string catalog, string schema, string table, string tableType) right) + { + int order = StringComparer.Ordinal.Compare(left.tableType, right.tableType); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.catalog, right.catalog); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.schema, right.schema); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.table, right.table); + } + async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPattern, string? schemaPattern, string? tablePattern, string? columnPattern, Dictionary>> catalogMap, CancellationToken cancellationToken) { - List<(RecordBatch Batch, bool IsNative, string? Catalog)> batches; - try + var batches = new List<(RecordBatch Batch, bool IsNative, string? Catalog)>(); + foreach (string? catalog in await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false)) { - batches = await ExecuteNativeShowColumnsAsync( - catalogPattern, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - // Object-not-found → leave the column info unpopulated (empty), matching - // the flat metadata methods and the JDBC reference driver. - return; + try + { + batches.AddRange(await ExecuteNativeShowColumnsAsync( + catalog, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false)); + } + catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) + { + // A catalog may disappear after enumeration. + } } var tablePositions = new Dictionary(); @@ -986,6 +1017,8 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter string? colType = isNative ? native!.String("TYPE_NAME", i) : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; + if (isNative && sourceCatalog != null && + !string.Equals(sourceCatalog, cat, StringComparison.OrdinalIgnoreCase)) continue; if (string.IsNullOrEmpty(colName)) continue; @@ -1153,6 +1186,22 @@ private static bool MatchesCatalogPattern(string? pattern, string catalog) TimeSpan.FromSeconds(1)); } + private async Task> ResolveGetObjectsCatalogsAsync( + string? pattern, CancellationToken cancellationToken) + { + if (pattern == null || (pattern.Length > 0 && pattern.All(character => character == '%'))) + return new string?[] { null }; + if (pattern == "") + return System.Array.Empty(); + if (pattern.IndexOfAny(new[] { '%', '_', '\\' }) < 0) + return new string?[] { pattern }; + + // SHOW scopes are identifiers; resolve LIKE patterns to concrete catalog names. + IReadOnlyList catalogs = await ((IGetObjectsDataProvider)this) + .GetCatalogsAsync(pattern, cancellationToken).ConfigureAwait(false); + return catalogs.Select(catalog => (string?)catalog).ToList(); + } + internal bool EnablePKFK => _enablePKFK; internal bool EnableMultipleCatalogSupport => _enableMultipleCatalogSupport; diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 5ec966a1..9969dcc2 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -314,6 +314,7 @@ public Task ExecuteQueryAsync( internal async Task ExecuteQueryAsync( CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation) { + IsNativeMetadataResult = false; if (_isMetadataCommand) { return await ExecuteMetadataCommandAsync(cancellationToken).ConfigureAwait(false); @@ -394,7 +395,6 @@ private async Task EnsureCatalogScopedAsync(CancellationToken cancellationToken) private async Task ExecuteQueryInternalAsync( CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation = null) { - IsNativeMetadataResult = false; // If the caller explicitly scoped this statement to a catalog, set the session's // current catalog first via USE CATALOG so a 2-level `schema`.`table` name // resolves. The server rejects catalog+session_id together, and SEA always uses a @@ -471,6 +471,8 @@ private async Task ExecuteQueryInternalAsync( _statementClosedByServer = true; } + // The server may fall back to SHOW even with the require-native header. + // Only the response manifest selects the decoder. IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult ?? initialNativeMetadata ?? false; @@ -1293,6 +1295,7 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT activity?.SetTag("sql_query", sql); var (batches, isNative) = await _connection.ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = isNative; if (isNative) return NativeMetadataResultBuilder.Build( batches, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); @@ -1349,6 +1352,7 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); batches = nativeResult.Batches; + IsNativeMetadataResult = nativeResult.IsNative; if (nativeResult.IsNative) return NativeMetadataResultBuilder.Build( batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, @@ -1439,6 +1443,7 @@ private async Task GetTablesAsync(CancellationToken cancellationTok var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); batches = nativeResult.Batches; + IsNativeMetadataResult = nativeResult.IsNative; if (nativeResult.IsNative) { var requestedTypes = _metadataTableTypes?.Split(',').Select(type => type.Trim()).ToArray(); @@ -1556,7 +1561,8 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo System.Array.Empty<(string, string, string, TableInfo)>()); } - if (columnBatches.Count > 0 && columnBatches.All(result => result.IsNative)) + IsNativeMetadataResult = columnBatches.Count > 0 && columnBatches.All(result => result.IsNative); + if (IsNativeMetadataResult) { return NativeMetadataResultBuilder.Build( columnBatches.Select(result => result.Batch).ToList(), @@ -1866,6 +1872,7 @@ private async Task GetPrimaryKeysAsyncNoThrow( var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); batches = nativeResult.Batches; + IsNativeMetadataResult = nativeResult.IsNative; if (nativeResult.IsNative) { return NativeMetadataResultBuilder.Build( @@ -1995,6 +2002,7 @@ private async Task GetCrossReferenceAsyncNoThrow( var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); batches = nativeResult.Batches; + IsNativeMetadataResult = nativeResult.IsNative; if (nativeResult.IsNative) { return NativeMetadataResultBuilder.Build( diff --git a/csharp/test/E2E/DatabricksTestConfiguration.cs b/csharp/test/E2E/DatabricksTestConfiguration.cs index 537e74f0..23c57f82 100644 --- a/csharp/test/E2E/DatabricksTestConfiguration.cs +++ b/csharp/test/E2E/DatabricksTestConfiguration.cs @@ -78,5 +78,8 @@ public class DatabricksTestConfiguration : SparkTestConfiguration [JsonPropertyName("protocol"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string Protocol { get; set; } = string.Empty; + + [JsonPropertyName("requireNativeMetadata"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] + public bool RequireNativeMetadata { get; set; } } } diff --git a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs index fe616441..13446a6d 100644 --- a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs +++ b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs @@ -19,6 +19,7 @@ using System.IO; using System.Text.Json; using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; using Apache.Arrow; using Apache.Arrow.Adbc; using Apache.Arrow.Adbc.Tests; @@ -84,7 +85,8 @@ private AdbcConnection CreateConnection(Dictionary? extraParams private async Task>> ReadMetadata(AdbcConnection connection, string command, string? catalog = null, string? schema = null, string? table = null, string? column = null, - string? tableTypes = null, bool escapeWildcards = false) + string? tableTypes = null, bool escapeWildcards = false, bool requireNative = false, + string? foreignTable = null) { var results = new List>(); using var stmt = connection.CreateStatement(); @@ -95,10 +97,21 @@ private async Task>> ReadMetadata(AdbcConnection if (table != null) stmt.SetOption(ApacheParameters.TableName, table); if (column != null) stmt.SetOption(ApacheParameters.ColumnName, column); if (tableTypes != null) stmt.SetOption(ApacheParameters.TableTypes, tableTypes); + if (foreignTable != null) + { + stmt.SetOption(ApacheParameters.ForeignCatalogName, catalog!); + stmt.SetOption(ApacheParameters.ForeignSchemaName, schema!); + stmt.SetOption(ApacheParameters.ForeignTableName, foreignTable); + } stmt.SqlQuery = command; var result = stmt.ExecuteQuery(); using var reader = result.Stream; + if (requireNative) + { + Assert.True(Assert.IsType(stmt).IsNativeMetadataResult, + $"{command} fell back to SHOW; this warehouse must return is_native_metadata_result: true."); + } while (true) { @@ -200,6 +213,31 @@ public async Task GetTables_TypesFilter_IsCaseSensitive() private static readonly Dictionary RestProtocol = new() { { DatabricksParameters.Protocol, "rest" } }; + [SkippableTheory] + [InlineData("GetCatalogs", "all_column_types", null, "TABLE_CAT", "main")] + [InlineData("GetSchemas", "all_column_types", null, "TABLE_SCHEM", TestSchema)] + [InlineData("GetTables", "all_column_types", null, "TABLE_NAME", "all_column_types")] + [InlineData("GetColumns", "all_column_types", null, "COLUMN_NAME", "c_int")] + [InlineData("GetPrimaryKeys", "cross_ref_customers", null, "TABLE_NAME", "cross_ref_customers")] + [InlineData("GetCrossReference", "cross_ref_customers", "cross_ref_orders", "PKTABLE_NAME", "cross_ref_customers")] + public async Task NativeMetadata_UsesNativeResponse( + string command, string table, string? foreignTable, string expectedColumn, string expectedValue) + { + SkipIfNotConfigured(); + Skip.IfNot(TestConfiguration.RequireNativeMetadata, + "Set requireNativeMetadata=true in the test configuration for a native-capable warehouse."); + + using var conn = CreateConnection(new Dictionary(RestProtocol) + { + [DatabricksParameters.EnableMultipleCatalogSupport] = "true", + [DatabricksParameters.EnablePKFK] = "true", + }); + var rows = await ReadMetadata(conn, command, TestCatalog, TestSchema, table, + requireNative: true, foreignTable: foreignTable); + + Assert.Contains(rows, row => row[expectedColumn] == expectedValue); + } + [SkippableFact] public async Task GetColumns_CatalogMatchAll_WithEscaping_ReturnsEmpty() { diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs new file mode 100644 index 00000000..b793e04a --- /dev/null +++ b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs @@ -0,0 +1,281 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using AdbcDrivers.HiveServer2.Spark; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Adbc.Tests.Metadata; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Moq; +using Moq.Protected; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class StatementExecutionGetObjectsCatalogPatternTests + { + private static readonly string[] s_catalogs = { "main", "marketing", "other", "foo_bar", "fooxbar" }; + + public static IEnumerable CatalogPatterns() + { + (string? Pattern, string[] Matches)[] patterns = + { + ("main", new[] { "main" }), + ("ma%", new[] { "main", "marketing" }), + ("ma_n", new[] { "main" }), + (null, s_catalogs), + ("%", s_catalogs), + ("%%", s_catalogs), + ("", System.Array.Empty()), + ("missing%", System.Array.Empty()), + ("MA%", new[] { "main", "marketing" }), + (@"foo\_bar", new[] { "foo_bar" }), + ("foo_bar", new[] { "foo_bar", "fooxbar" }), + }; + foreach (int mode in new[] { 0, 1, 2 }) + { + foreach (AdbcConnection.GetObjectsDepth depth in new[] + { AdbcConnection.GetObjectsDepth.Tables, AdbcConnection.GetObjectsDepth.All }) + { + foreach (var (pattern, matches) in patterns) + yield return new object?[] { mode, depth, pattern, matches }; + } + } + } + + [Theory] + [MemberData(nameof(CatalogPatterns))] + public async Task GetObjects_ResolvesCatalogPatterns( + int mode, AdbcConnection.GetObjectsDepth depth, string? pattern, string[] expectedCatalogs) + { + List statements = new List(); + using HttpClient http = CreateHttpClient(mode, statements); + using StatementExecutionConnection connection = CreateConnection(http); + using IArrowArrayStream stream = connection.GetObjects(depth, pattern, "default", "t1", null, null); + using RecordBatch? batch = await stream.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + List catalogs = GetObjectsParser.ParseCatalog(batch, null); + Assert.Equal(expectedCatalogs.OrderBy(name => name), catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); + foreach (AdbcCatalog catalog in catalogs) + { + AdbcDbSchema schema = Assert.Single(catalog.DbSchemas!); + Assert.Equal("default", schema.Name); + AdbcTable table = Assert.Single(schema.Tables!); + Assert.Equal("t1", table.Name); + if (depth == AdbcConnection.GetObjectsDepth.All) + { + AdbcColumn column = Assert.Single(table.Columns!); + Assert.Equal("a", column.Name); + Assert.Equal(1, column.OrdinalPosition); + } + } + + if (pattern == "") + Assert.Empty(statements); + if (pattern == "main") + Assert.Single(statements, sql => sql.StartsWith("SHOW CATALOGS", StringComparison.Ordinal)); + } + + [Theory] + [InlineData("GetCatalogs", false)] + [InlineData("GetCatalogs", true)] + [InlineData("GetSchemas", false)] + [InlineData("GetSchemas", true)] + [InlineData("GetTables", false)] + [InlineData("GetTables", true)] + [InlineData("GetColumns", false)] + [InlineData("GetColumns", true)] + public void MetadataCommand_RecordsResponseFormat(string command, bool native) + { + using HttpClient http = CreateHttpClient(native ? 1 : 0, new List()); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SetOption(ApacheParameters.SchemaName, "default"); + statement.SetOption(ApacheParameters.TableName, "t1"); + statement.SqlQuery = command; + QueryResult result = statement.ExecuteQuery(); + using IArrowArrayStream? stream = result.Stream; + + Assert.Equal(native, statement.IsNativeMetadataResult); + } + + [Fact] + public void MetadataCommand_ResetsNativeFlagForSyntheticResults() + { + using HttpClient http = CreateHttpClient(1, new List()); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SqlQuery = "GetTables"; + using IArrowArrayStream? native = statement.ExecuteQuery().Stream; + Assert.True(statement.IsNativeMetadataResult); + + statement.SetOption(ApacheParameters.TableTypes, ""); + using IArrowArrayStream? empty = statement.ExecuteQuery().Stream; + Assert.False(statement.IsNativeMetadataResult); + } + + private static StatementExecutionConnection CreateConnection(HttpClient http) + { + Dictionary properties = new Dictionary + { + [SparkParameters.HostName] = "test.databricks.com", + [DatabricksParameters.WarehouseId] = "wh-1", + [SparkParameters.AccessToken] = "token", + }; + return new StatementExecutionConnection(properties, http); + } + + private static HttpClient CreateHttpClient(int mode, List statements) + { + Mock handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => + { + if (request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true) + return Response("{\"session_id\":\"s1\"}"); + if (request.Method != HttpMethod.Post) + return new HttpResponseMessage(HttpStatusCode.OK); + + using JsonDocument json = JsonDocument.Parse( + request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); + string sql = json.RootElement.GetProperty("statement").GetString()!; + statements.Add(sql); + string operation = request.Headers.GetValues("x-databricks-metadata-operation-type").Single(); + bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables"); + string? requestedCatalog = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs" + ? null : s_catalogs.Single(catalog => sql.Contains($"`{catalog}`")); + string[] catalogs = requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; + if (native && requestedCatalog != null && operation is "GetTables" or "GetColumns") + catalogs = catalogs.Concat(new[] { requestedCatalog == "other" ? "main" : "other" }).ToArray(); + + using RecordBatch batch = CreateBatch(operation, native, catalogs, requestedCatalog); + using MemoryStream raw = new MemoryStream(); + using (ArrowStreamWriter writer = new ArrowStreamWriter(raw, batch.Schema)) + { + writer.WriteRecordBatch(batch); + writer.WriteEnd(); + } + return Response(JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = native, + total_row_count = batch.Length, + schema = new + { + column_count = batch.ColumnCount, + columns = batch.Schema.FieldsList.Select((field, position) => new + { + name = field.Name, + position, + type_name = field.DataType.TypeId == ArrowTypeId.String ? "STRING" + : field.DataType.TypeId == ArrowTypeId.Int16 ? "SMALLINT" : "INT", + }).ToArray(), + }, + }, + result = new { attachment = raw.ToArray() }, + })); + }); + return new HttpClient(handler.Object); + } + + private static HttpResponseMessage Response(string body) + => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + + private static RecordBatch CreateBatch(string operation, bool native, string[] catalogs, string? requestedCatalog) + { + Schema schema; + if (native) + { + schema = operation switch + { + "GetCatalogs" => MetadataSchemaFactory.CreateCatalogsSchema(), + "GetSchemas" => MetadataSchemaFactory.CreateSchemasSchema(), + "GetTables" => MetadataSchemaFactory.CreateTablesSchema(), + _ => new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23), null), + }; + } + else + { + string[] names = operation switch + { + "GetCatalogs" => new[] { "catalog" }, + "GetSchemas" => requestedCatalog == null ? new[] { "databaseName", "catalog" } : new[] { "databaseName" }, + "GetTables" => new[] { "catalogName", "namespace", "tableName", "tableType" }, + _ => new[] { "catalogName", "namespace", "tableName", "col_name", "columnType", "isNullable" }, + }; + schema = new Schema(names.Select(name => new Field(name, StringType.Default, true)), null); + } + + IArrowArray[] arrays = schema.FieldsList.Select(field => + { + if (field.DataType.TypeId == ArrowTypeId.String) + { + StringArray.Builder strings = new StringArray.Builder(); + foreach (string catalog in catalogs) + { + string? value = field.Name switch + { + "TABLE_CAT" or "catalog" or "catalogName" => catalog, + "TABLE_CATALOG" => requestedCatalog == null ? catalog : null, + "TABLE_SCHEM" or "databaseName" or "namespace" => "default", + "TABLE_NAME" or "tableName" => "t1", + "TABLE_TYPE" or "tableType" => "TABLE", + "COLUMN_NAME" or "col_name" => "a", + "TYPE_NAME" or "columnType" => "INT", + "isNullable" => "true", + _ => null, + }; + if (value == null) strings.AppendNull(); else strings.Append(value); + } + return (IArrowArray)strings.Build(); + } + int number = field.Name == "NULLABLE" ? 1 : field.Name == "DATA_TYPE" ? 4 : 0; + return field.DataType.TypeId switch + { + ArrowTypeId.Int16 => (IArrowArray)new Int16Array.Builder().AppendRange( + Enumerable.Repeat((short)number, catalogs.Length)).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().AppendRange( + Enumerable.Repeat(number, catalogs.Length)).Build(), + _ => throw new InvalidOperationException($"Unexpected metadata type: {field.DataType}"), + }; + }).ToArray(); + return new RecordBatch(schema, arrays, catalogs.Length); + } + } +} From 4906708b846cd57107701f7c660ba7f6e0948ccf Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 03:43:25 +0000 Subject: [PATCH 13/22] fix(csharp): propagate native metadata validation errors Signed-off-by: Vu Anh Phung --- .../StatementExecutionConnection.cs | 53 ++++++++++--------- .../StatementExecutionNativeCatalogTests.cs | 35 +++++++++++- 2 files changed, 61 insertions(+), 27 deletions(-) diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 8b3d8d54..592f1c34 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -1124,40 +1124,43 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc if (sourceCatalog == null) continue; string sql = new ShowColumnsCommand( sourceCatalog, schemaPattern, tablePattern, columnPattern).Build(); + List batches; + bool isNative; try { - var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + (batches, isNative) = await ExecuteNativeMetadataSqlAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - foreach (var columns in batches) + } + catch + { + // Skip catalogs we can't access (permission errors) + continue; + } + foreach (var columns in batches) + { + if (!isNative) { - if (!isNative) - { - results.Add((columns, false, sourceCatalog)); - continue; - } + results.Add((columns, false, sourceCatalog)); + continue; + } - // Native GetColumns can treat catalog names as LIKE patterns. - var nativeColumns = new NativeMetadataColumns( - columns, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); - int start = -1; - for (int row = 0; row <= columns.Length; row++) + // Native GetColumns can treat catalog names as LIKE patterns. + var nativeColumns = new NativeMetadataColumns( + columns, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); + int start = -1; + for (int row = 0; row <= columns.Length; row++) + { + string? rowCatalog = row < columns.Length ? nativeColumns.String("TABLE_CAT", row) : null; + bool matches = row < columns.Length && (rowCatalog == null || + string.Equals(rowCatalog, sourceCatalog, StringComparison.OrdinalIgnoreCase)); + if (matches && start < 0) start = row; + if (!matches && start >= 0) { - string? rowCatalog = row < columns.Length ? nativeColumns.String("TABLE_CAT", row) : null; - bool matches = row < columns.Length && (rowCatalog == null || - string.Equals(rowCatalog, sourceCatalog, StringComparison.OrdinalIgnoreCase)); - if (matches && start < 0) start = row; - if (!matches && start >= 0) - { - results.Add((columns.Slice(start, row - start), true, sourceCatalog)); - start = -1; - } + results.Add((columns.Slice(start, row - start), true, sourceCatalog)); + start = -1; } } } - catch - { - // Skip catalogs we can't access (permission errors) - } } } return results; diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index e8821319..93745360 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -87,12 +87,13 @@ private static byte[] ArrowNativeColumns(Field[] fields) return raw.ToArray(); } - private static HttpClient CreateHttpClient(bool overlappingNativeColumns = false) + private static HttpClient CreateHttpClient(bool overlappingNativeColumns = false, Field[]? columnFields = null) { byte[] catalogs = overlappingNativeColumns ? ArrowStrings("TABLE_CAT", "foo_bar", "fooxbar") : ArrowStrings("TABLE_CAT", "main", "other"); - Field[] nativeColumnFields = MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); + Field[] nativeColumnFields = columnFields ?? + MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); byte[] columns = overlappingNativeColumns ? ArrowNativeColumns(nativeColumnFields) : ArrowStrings("col_name", "a"); @@ -199,6 +200,36 @@ public async Task ColumnFanout_KeepsTheQueriedCatalog() Assert.Equal("other", batches[1].Catalog); } + [Theory] + [InlineData("count", "expected 23 columns, found 22")] + [InlineData("name", "missing TABLE_CAT")] + [InlineData("type", "unexpected type or duplicate TABLE_SCHEM")] + public async Task ColumnFanout_MalformedNativeColumns_PropagatesValidationError( + string invalidSchema, string expectedError) + { + Field[] fields = MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); + switch (invalidSchema) + { + case "count": + fields = fields.Skip(1).ToArray(); + break; + case "name": + fields[0] = new Field("UNKNOWN", fields[0].DataType, true); + break; + case "type": + fields[1] = new Field(fields[1].Name, Int32Type.Default, true); + break; + } + using var http = CreateHttpClient(overlappingNativeColumns: true, columnFields: fields); + using var connection = CreateConnection(http); + + var exception = await Assert.ThrowsAsync(() => + connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); + + Assert.StartsWith("Invalid native GetColumns result:", exception.Message); + Assert.Contains(expectedError, exception.Message); + } + [Fact] public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() { From cf939e4459ffd82b3b2fa5f009f2133fb4dcf8ad Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 04:09:27 +0000 Subject: [PATCH 14/22] refactor(csharp): share native and SHOW metadata decoding Signed-off-by: Vu Anh Phung --- .../StatementExecution/MetadataRowReader.cs | 321 +++++++++++++++ .../NativeMetadataResultBuilder.cs | 6 +- .../StatementExecutionConnection.cs | 367 +++--------------- .../StatementExecutionStatement.cs | 207 +++------- .../MetadataRowReaderTests.cs | 173 +++++++++ ...tExecutionGetObjectsCatalogPatternTests.cs | 33 +- ...entExecutionMetadataObjectNotFoundTests.cs | 2 +- .../StatementExecutionNativeCatalogTests.cs | 27 +- 8 files changed, 659 insertions(+), 477 deletions(-) create mode 100644 csharp/src/StatementExecution/MetadataRowReader.cs create mode 100644 csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs new file mode 100644 index 00000000..50ad0660 --- /dev/null +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -0,0 +1,321 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal sealed class MetadataBatches + { + internal MetadataBatches(List batches, bool isNative, + string? sourceCatalog = null, bool requireExactCatalog = false) + { + Batches = batches; + IsNative = isNative; + SourceCatalog = sourceCatalog; + RequireExactCatalog = requireExactCatalog; + } + + internal List Batches { get; } + internal bool IsNative { get; } + internal string? SourceCatalog { get; } + internal bool RequireExactCatalog { get; } + } + + internal sealed class ColumnMetadataResult + { + internal ColumnMetadataResult(IReadOnlyList results) + { + Batches = results.SelectMany(result => result.Batches).ToList(); + SourceCatalogs = results.SelectMany(result => + Enumerable.Repeat(result.SourceCatalog, result.Batches.Count)).ToList(); + IsNative = Batches.Count > 0 && + results.Where(result => result.Batches.Count > 0).All(result => result.IsNative); + Rows = MetadataRowReader.Columns(results).ToList(); + } + + internal IReadOnlyList Batches { get; } + internal IReadOnlyList SourceCatalogs { get; } + internal bool IsNative { get; } + internal IReadOnlyList Rows { get; } + } + + internal readonly struct SchemaRow + { + internal SchemaRow(string catalog, string schema) + { + Catalog = catalog; + Schema = schema; + } + + internal string Catalog { get; } + internal string Schema { get; } + } + + internal readonly struct TableRow + { + internal TableRow(string catalog, string schema, string table, string tableType, string remarks) + { + Catalog = catalog; + Schema = schema; + Table = table; + TableType = tableType; + Remarks = remarks; + } + + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string TableType { get; } + internal string Remarks { get; } + } + + internal readonly struct ColumnRow + { + internal ColumnRow(string? catalog, string? schema, string? table, string name, + string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement) + { + Catalog = catalog; + Schema = schema; + Table = table; + Name = name; + TypeName = typeName; + Nullable = nullable; + Ordinal = ordinal; + Default = columnDefault; + IsAutoIncrement = isAutoIncrement; + } + + internal string? Catalog { get; } + internal string? Schema { get; } + internal string? Table { get; } + internal string Name { get; } + internal string TypeName { get; } + internal bool Nullable { get; } + internal int Ordinal { get; } + internal string? Default { get; } + internal bool IsAutoIncrement { get; } + } + + internal static class MetadataRowReader + { + internal static IEnumerable Catalogs(MetadataBatches result, string? pattern = null) + { + foreach (var batch in result.Batches) + { + var native = result.IsNative + ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) + : null; + var show = result.IsNative ? null : TryGetColumn(batch, "catalog"); + if (!result.IsNative && show == null) continue; + for (int row = 0; row < batch.Length; row++) + { + string? catalog = result.IsNative ? native!.String("TABLE_CAT", row) : String(show, row); + if (catalog != null && MatchesCatalogPattern(pattern, catalog)) + yield return catalog; + } + } + } + + internal static IEnumerable Schemas(MetadataBatches result, string? catalog) + { + foreach (var batch in result.Batches) + { + if (result.IsNative) + { + var native = new NativeMetadataColumns( + batch, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); + for (int row = 0; row < batch.Length; row++) + { + string? schema = native.String("TABLE_SCHEM", row); + string? rowCatalog = native.String("TABLE_CATALOG", row) ?? catalog; + if (schema != null && MatchesCatalog(catalog, rowCatalog)) + yield return new SchemaRow(rowCatalog ?? "", schema); + } + continue; + } + + // Scoped SHOW SCHEMAS omits the catalog column. + var schemas = batch.Column(0) as StringArray; + var catalogs = catalog == null ? batch.Column(1) as StringArray : null; + if (schemas == null) continue; + for (int row = 0; row < batch.Length; row++) + { + string? schema = String(schemas, row); + if (schema != null) + yield return new SchemaRow(String(catalogs, row) ?? catalog ?? "", schema); + } + } + } + + internal static List Tables( + MetadataBatches result, string? catalog, IReadOnlyCollection? tableTypes = null) + { + var rows = new List(); + foreach (var batch in result.Batches) + { + var native = result.IsNative + ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables) + : null; + var catalogs = result.IsNative ? null : TryGetColumn(batch, "catalogName"); + var schemas = result.IsNative ? null : TryGetColumn(batch, "namespace"); + var tables = result.IsNative ? null : TryGetColumn(batch, "tableName"); + var types = result.IsNative ? null : TryGetColumn(batch, "tableType"); + var remarks = result.IsNative ? null : TryGetColumn(batch, "remarks"); + if (!result.IsNative && (catalogs == null || schemas == null || tables == null)) continue; + + for (int row = 0; row < batch.Length; row++) + { + string? rowCatalog = result.IsNative ? native!.String("TABLE_CAT", row) ?? catalog : String(catalogs, row); + string? schema = result.IsNative ? native!.String("TABLE_SCHEM", row) : String(schemas, row); + string? table = result.IsNative ? native!.String("TABLE_NAME", row) : String(tables, row); + if ((!result.IsNative && rowCatalog == null) || schema == null || table == null) continue; + if (result.IsNative && !MatchesCatalog(catalog, rowCatalog)) continue; + string type = NativeMetadataResultBuilder.DefaultTableType( + result.IsNative ? native!.String("TABLE_TYPE", row) : String(types, row)); + if (tableTypes != null && !tableTypes.Contains(type)) continue; + rows.Add(new TableRow(rowCatalog ?? "", schema, table, type, + (result.IsNative ? native!.String("REMARKS", row) : String(remarks, row)) ?? "")); + } + } + // Thrift orders tables by type, catalog, schema, then name. + if (result.IsNative) rows.Sort(CompareTables); + return rows; + } + + internal static IEnumerable Columns(IEnumerable results) + { + var positions = new Dictionary(); + foreach (var result in results) + { + foreach (var batch in result.Batches) + { + var rows = result.IsNative ? NativeColumns(batch, result) : ShowColumns(batch, positions); + foreach (var row in rows) + { + string key = $"{row.Catalog}.{row.Schema}.{row.Table}"; + positions.TryGetValue(key, out int position); + positions[key] = position + 1; + yield return row; + } + } + } + } + + private static IEnumerable NativeColumns(RecordBatch batch, MetadataBatches result) + { + var columns = new NativeMetadataColumns( + batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); + for (int row = 0; row < batch.Length; row++) + { + string? catalog = columns.String("TABLE_CAT", row) ?? result.SourceCatalog; + // A quoted catalog can still expand as a native LIKE pattern. + if (result.RequireExactCatalog && !MatchesCatalog(result.SourceCatalog, catalog)) continue; + string? name = columns.String("COLUMN_NAME", row); + string? typeName = columns.String("TYPE_NAME", row); + if (name == null || typeName == null) continue; + // Both decoders expose zero-based ordinals; GetObjects adds its required offset. + yield return new ColumnRow(catalog, columns.String("TABLE_SCHEM", row), + columns.String("TABLE_NAME", row), name, typeName, columns.Integer("NULLABLE", row) == 1, + checked((int)(columns.Integer("ORDINAL_POSITION", row) ?? 0)), + columns.String("COLUMN_DEF", row), + string.Equals(columns.String("IS_AUTO_INCREMENT", row), "YES", StringComparison.OrdinalIgnoreCase)); + } + } + + private static IEnumerable ShowColumns(RecordBatch batch, Dictionary positions) + { + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var names = TryGetColumn(batch, "col_name"); + var types = TryGetColumn(batch, "columnType"); + var nullable = TryGetColumn(batch, "isNullable"); + if (names == null || types == null) yield break; + for (int row = 0; row < batch.Length; row++) + { + string? name = String(names, row); + string? typeName = String(types, row); + if (name == null || typeName == null) continue; + string? catalog = String(catalogs, row); + string? schema = String(schemas, row); + string? table = String(tables, row); + positions.TryGetValue($"{catalog}.{schema}.{table}", out int ordinal); + yield return new ColumnRow(catalog, schema, table, name, typeName, + !string.Equals(String(nullable, row), "false", StringComparison.OrdinalIgnoreCase), + ordinal, null, false); + } + } + + internal static int CompareTables(TableRow left, TableRow right) + { + int order = StringComparer.Ordinal.Compare(left.TableType, right.TableType); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.Catalog, right.Catalog); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.Schema, right.Schema); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.Table, right.Table); + } + + internal static bool MatchesCatalog(string? requested, string? actual) + => requested == null || string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase); + + private static string? String(StringArray? values, int row) + => values == null || values.IsNull(row) ? null : values.GetString(row); + + private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray + { + try + { + return batch.Column(name) as T; + } + catch (ArgumentOutOfRangeException) + { + return null; + } + } + + private static bool MatchesCatalogPattern(string? pattern, string catalog) + { + if (pattern == null) return true; + var regex = new StringBuilder("^"); + bool escaped = false; + foreach (char character in pattern) + { + if (!escaped && character == '\\') + { + escaped = true; + continue; + } + regex.Append(!escaped && character == '%' ? ".*" + : !escaped && character == '_' ? "." + : Regex.Escape(character.ToString())); + escaped = false; + } + if (escaped) regex.Append(Regex.Escape("\\")); + regex.Append('$'); + return Regex.IsMatch(catalog, regex.ToString(), RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + } + } +} diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index b8a431a9..4958d185 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -31,7 +31,8 @@ internal static QueryResult Build( IReadOnlyList batches, Schema schema, MetadataOperation operation, string? requestedCatalog = null, IReadOnlyCollection? tableTypes = null, string? parentCatalog = null, string? parentSchema = null, string? parentTable = null, - IReadOnlyList? sourceCatalogs = null, bool requireExactCatalog = false) + IReadOnlyList? sourceCatalogs = null, bool requireExactCatalog = false, + bool requireExactSourceCatalog = false) { // C# exposes BASE_TYPE_NAME after the 23 Thrift GetColumns fields. int sourceColumns = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); @@ -52,6 +53,9 @@ internal static QueryResult Build( for (int row = 0; row < batch.Length; row++) { + if (operation == MetadataOperation.GetColumns && requireExactSourceCatalog && + !MetadataRowReader.MatchesCatalog(sourceCatalog, columns.String("TABLE_CAT", row) ?? sourceCatalog)) + continue; // Native metadata can expand wildcard characters in a quoted catalog. if (catalogField != null && requestedCatalog != null && (operation == MetadataOperation.GetTables || requireExactCatalog) && diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 592f1c34..68239949 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -18,8 +18,6 @@ using System.Collections.Generic; using System.Linq; using System.Net.Http; -using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.Http; @@ -712,13 +710,13 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string using var cts = CreateMetadataTimeoutCts(); // Pass catalog through with SPARK→null normalization, matching Thrift - // which sends catalog as-is to the server. ExecuteNativeShowColumnsAsync + // which sends catalog as-is to the server. ReadColumnsAsync // handles null by iterating all catalogs. string? resolvedCatalog = DatabricksConnection.HandleSparkCatalog(catalog); - List<(RecordBatch Batch, bool IsNative, string? Catalog)> batches; + ColumnMetadataResult columns; try { - batches = ExecuteNativeShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) + columns = ReadColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) .GetAwaiter().GetResult(); } catch (Exception ex) when (cts.IsCancellationRequested && ex is not TimeoutException) @@ -730,32 +728,11 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string } var fields = new List(); - foreach (var (batch, isNative, _) in batches) + foreach (var column in columns.Rows) { - var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; - var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - - if (!isNative && (colNameArray == null || columnTypeArray == null)) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (isNative - ? native!.String("COLUMN_NAME", i) == null || native.String("TYPE_NAME", i) == null - : colNameArray!.IsNull(i) || columnTypeArray!.IsNull(i)) continue; - - string colName = isNative ? native!.String("COLUMN_NAME", i)! : colNameArray!.GetString(i); - string colType = isNative ? native!.String("TYPE_NAME", i)! : columnTypeArray!.GetString(i); - bool nullable = isNative - ? native!.Integer("NULLABLE", i) == 1 - : isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - short typeCode = ColumnMetadataHelper.GetDataTypeCode(colType); - IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, colType, false, null, null); - fields.Add(new Field(colName, arrowType, nullable)); - } + short typeCode = ColumnMetadataHelper.GetDataTypeCode(column.TypeName); + IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, column.TypeName, false, null, null); + fields.Add(new Field(column.Name, arrowType, column.Nullable)); } activity?.SetTag("result_fields", fields.Count); @@ -771,11 +748,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin return System.Array.Empty(); string sql = new ShowCatalogsCommand(catalogPattern).Build(); - List batches; - bool isNative; + MetadataBatches result; try { - (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + result = await ExecuteMetadataCommandAsync( sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) @@ -784,21 +760,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin // flat metadata methods and the JDBC reference driver). return System.Array.Empty(); } - var result = new List(); - foreach (var batch in batches) - { - var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) : null; - var catalogArray = isNative ? null : TryGetColumn(batch, "catalog"); - if (!isNative && catalogArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - string? name = isNative ? native!.String("TABLE_CAT", i) - : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - if (name != null && MatchesCatalogPattern(catalogPattern, name)) - result.Add(name); - } - } - return result; + return MetadataRowReader.Catalogs(result, catalogPattern).ToList(); } async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) @@ -816,11 +778,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin { string sql = new ShowSchemasCommand(catalog, schemaPattern).Build(); - List batches; - bool isNative; + MetadataBatches result; try { - (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + result = await ExecuteMetadataCommandAsync( sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) @@ -828,50 +789,8 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin return System.Array.Empty<(string, string)>(); } - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showSchemasInAllCatalogs = catalog == null; - - var result = new List<(string, string)>(); - foreach (var batch in batches) - { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (isNative) - { - var native = new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); - for (int i = 0; i < batch.Length; i++) - { - string? name = native.String("TABLE_SCHEM", i); - string? rowCatalog = native.String("TABLE_CATALOG", i) ?? catalog; - if (name != null && (catalog == null || - string.Equals(catalog, rowCatalog, StringComparison.OrdinalIgnoreCase))) - result.Add((rowCatalog ?? "", name)); - } - continue; - } - else if (showSchemasInAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - string rowCatalog = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalog ?? ""; - result.Add((rowCatalog, schemaArray.GetString(i))); - } - } - return result; + return MetadataRowReader.Schemas(result, catalog) + .Select(row => (row.Catalog, row.Schema)).ToList(); } async Task> IGetObjectsDataProvider.GetTablesAsync( @@ -888,86 +807,34 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin } IReadOnlyList catalogs = await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false); - var result = new List<(string catalog, string schema, string table, string tableType)>(); + var result = new List(); foreach (string? catalog in catalogs) { result.AddRange(await GetTablesInCatalogAsync( catalog, schemaPattern, tableNamePattern, tableTypes, cancellationToken).ConfigureAwait(false)); } if (catalogs.Count > 1) - result.Sort(CompareTables); - return result; + result.Sort(MetadataRowReader.CompareTables); + return result.Select(row => (row.Catalog, row.Schema, row.Table, row.TableType)).ToList(); } - private async Task> GetTablesInCatalogAsync( + private async Task> GetTablesInCatalogAsync( string? catalog, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes, CancellationToken cancellationToken) { string sql = new ShowTablesCommand(catalog, schemaPattern, tableNamePattern).Build(); - List batches; - bool isNative; + MetadataBatches result; try { - (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + result = await ExecuteMetadataCommandAsync( sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { - return System.Array.Empty<(string, string, string, string)>(); - } - var result = new List<(string catalog, string schema, string table, string tableType)>(); - foreach (var batch in batches) - { - var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables) : null; - var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); - var tableArray = isNative ? null : TryGetColumn(batch, "tableName"); - var tableTypeArray = isNative ? null : TryGetColumn(batch, "tableType"); - - if (!isNative && (catalogArray == null || schemaArray == null || tableArray == null)) continue; - - for (int i = 0; i < batch.Length; i++) - { - string? cat = isNative ? native!.String("TABLE_CAT", i) ?? catalog - : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - string? sch = isNative ? native!.String("TABLE_SCHEM", i) - : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); - string? tbl = isNative ? native!.String("TABLE_NAME", i) - : tableArray!.IsNull(i) ? null : tableArray.GetString(i); - if ((!isNative && cat == null) || sch == null || tbl == null) continue; - cat ??= ""; - if (isNative && catalog != null && - !string.Equals(catalog, cat, StringComparison.OrdinalIgnoreCase)) continue; - - string tableType = NativeMetadataResultBuilder.DefaultTableType(isNative - ? native!.String("TABLE_TYPE", i) - : tableTypeArray == null || tableTypeArray.IsNull(i) ? null : tableTypeArray.GetString(i)); - - // Empty (non-null) tableTypes filters to NO types (zero rows), - // matching databricks-jdbc. A null tableTypes means "all types" - // and skips this filter entirely. See METADATA-035. - if (tableTypes != null && !tableTypes.Contains(tableType)) - continue; - - result.Add((cat, sch, tbl, tableType)); - } + return System.Array.Empty(); } - if (isNative) - result.Sort(CompareTables); - return result; - } - - private static int CompareTables( - (string catalog, string schema, string table, string tableType) left, - (string catalog, string schema, string table, string tableType) right) - { - int order = StringComparer.Ordinal.Compare(left.tableType, right.tableType); - if (order != 0) return order; - order = StringComparer.Ordinal.Compare(left.catalog, right.catalog); - if (order != 0) return order; - order = StringComparer.Ordinal.Compare(left.schema, right.schema); - return order != 0 ? order : StringComparer.Ordinal.Compare(left.table, right.table); + return MetadataRowReader.Tables(result, catalog, tableTypes); } async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPattern, string? schemaPattern, @@ -975,13 +842,14 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - var batches = new List<(RecordBatch Batch, bool IsNative, string? Catalog)>(); + var columns = new List(); foreach (string? catalog in await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false)) { try { - batches.AddRange(await ExecuteNativeShowColumnsAsync( - catalog, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false)); + var result = await ReadColumnsAsync(catalog, schemaPattern, tablePattern, + columnPattern, cancellationToken, requireExactCatalog: true).ConfigureAwait(false); + columns.AddRange(result.Rows); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -989,87 +857,26 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter } } - var tablePositions = new Dictionary(); - - foreach (var (batch, isNative, sourceCatalog) in batches) + foreach (var column in columns) { - var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; - var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); - var tableNameArray = isNative ? null : TryGetColumn(batch, "tableName"); - var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - - if (!isNative && (catalogArray == null || schemaArray == null || tableNameArray == null || - colNameArray == null || columnTypeArray == null)) continue; - - for (int i = 0; i < batch.Length; i++) - { - string? cat = isNative ? native!.String("TABLE_CAT", i) ?? sourceCatalog ?? catalogPattern - : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - string? sch = isNative ? native!.String("TABLE_SCHEM", i) - : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); - string? tbl = isNative ? native!.String("TABLE_NAME", i) - : tableNameArray!.IsNull(i) ? null : tableNameArray.GetString(i); - string? colName = isNative ? native!.String("COLUMN_NAME", i) - : colNameArray!.IsNull(i) ? null : colNameArray.GetString(i); - string? colType = isNative ? native!.String("TYPE_NAME", i) - : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); - if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; - if (isNative && sourceCatalog != null && - !string.Equals(sourceCatalog, cat, StringComparison.OrdinalIgnoreCase)) continue; - - if (string.IsNullOrEmpty(colName)) continue; - - string tableKey = $"{cat}.{sch}.{tbl}"; - if (!tablePositions.ContainsKey(tableKey)) - tablePositions[tableKey] = 1; - int position = isNative - ? checked((int)(native!.Integer("ORDINAL_POSITION", i) ?? 0) + 1) - : tablePositions[tableKey]++; - - bool nullable = isNative - ? native!.Integer("NULLABLE", i) == 1 - : isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - if (catalogMap.TryGetValue(cat, out var schemaMap) - && schemaMap.TryGetValue(sch, out var tableMap) - && tableMap.TryGetValue(tbl, out var tableInfo)) - { - ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfo, colName, colType, position, nullable, - columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, - isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTO_INCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); - } - } - } - } - - private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray - { - try - { - return batch.Column(name) as T; - } - catch (ArgumentOutOfRangeException) - { - return null; + if (column.Catalog == null || column.Schema == null || column.Table == null || + string.IsNullOrEmpty(column.Name)) continue; + if (catalogMap.TryGetValue(column.Catalog, out var schemaMap) + && schemaMap.TryGetValue(column.Schema, out var tableMap) + && tableMap.TryGetValue(column.Table, out var tableInfo)) + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfo, column.Name, column.TypeName, checked(column.Ordinal + 1), column.Nullable, + columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); } } internal async Task> ExecuteMetadataSqlAsync(string sql, CancellationToken cancellationToken = default) { - var (batches, _) = await ExecuteMetadataSqlCoreAsync(sql, null, cancellationToken).ConfigureAwait(false); - return batches; + var result = await ExecuteMetadataCommandAsync(sql, null, cancellationToken).ConfigureAwait(false); + return result.Batches; } - internal Task<(List Batches, bool IsNative)> ExecuteNativeMetadataSqlAsync( - string sql, MetadataOperation operation, CancellationToken cancellationToken) - => ExecuteMetadataSqlCoreAsync(sql, operation, cancellationToken); - - private async Task<(List Batches, bool IsNative)> ExecuteMetadataSqlCoreAsync( + internal async Task ExecuteMetadataCommandAsync( string sql, MetadataOperation? operation, CancellationToken cancellationToken) { var batches = new List(); @@ -1078,7 +885,7 @@ internal async Task> ExecuteMetadataSqlAsync(string sql, Cance var result = await stmt.ExecuteQueryAsync( cancellationToken, isMetadataExecution: true, metadataOperation: operation).ConfigureAwait(false); using var stream = result.Stream; - if (stream == null) return (batches, stmt.IsNativeMetadataResult); + if (stream == null) return new MetadataBatches(batches, stmt.IsNativeMetadataResult); while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -1086,7 +893,7 @@ internal async Task> ExecuteMetadataSqlAsync(string sql, Cance if (batch == null) break; batches.Add(batch); } - return (batches, stmt.IsNativeMetadataResult); + return new MetadataBatches(batches, stmt.IsNativeMetadataResult); } internal List ExecuteMetadataSql(string sql, CancellationToken cancellationToken = default) @@ -1095,98 +902,42 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc } // SHOW COLUMNS IN ALL CATALOGS is unavailable, so fan out over catalogs. - internal async Task> ExecuteNativeShowColumnsAsync( + internal async Task ReadColumnsAsync( string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, - CancellationToken cancellationToken) + CancellationToken cancellationToken, bool requireExactCatalog = false) { - var results = new List<(RecordBatch, bool, string?)>(); + var results = new List(); if (catalog != null) { string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); - var (batches, isNative) = await ExecuteNativeMetadataSqlAsync( + var result = await ExecuteMetadataCommandAsync( sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.AddRange(batches.Select(batch => (batch, isNative, (string?)catalog))); - return results; + results.Add(new MetadataBatches(result.Batches, result.IsNative, catalog, requireExactCatalog)); + return new ColumnMetadataResult(results); } string catalogsSql = new ShowCatalogsCommand(null).Build(); - var (catalogBatches, catalogsAreNative) = await ExecuteNativeMetadataSqlAsync( + var catalogs = await ExecuteMetadataCommandAsync( catalogsSql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); - foreach (var batch in catalogBatches) + foreach (string sourceCatalog in MetadataRowReader.Catalogs(catalogs)) { - var native = catalogsAreNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) : null; - var catalogArray = catalogsAreNative ? null : TryGetColumn(batch, "catalog"); - if (!catalogsAreNative && catalogArray == null) continue; - for (int i = 0; i < batch.Length; i++) + string sql = new ShowColumnsCommand( + sourceCatalog, schemaPattern, tablePattern, columnPattern).Build(); + MetadataBatches result; + try { - string? sourceCatalog = catalogsAreNative ? native!.String("TABLE_CAT", i) - : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - if (sourceCatalog == null) continue; - string sql = new ShowColumnsCommand( - sourceCatalog, schemaPattern, tablePattern, columnPattern).Build(); - List batches; - bool isNative; - try - { - (batches, isNative) = await ExecuteNativeMetadataSqlAsync( - sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - } - catch - { - // Skip catalogs we can't access (permission errors) - continue; - } - foreach (var columns in batches) - { - if (!isNative) - { - results.Add((columns, false, sourceCatalog)); - continue; - } - - // Native GetColumns can treat catalog names as LIKE patterns. - var nativeColumns = new NativeMetadataColumns( - columns, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); - int start = -1; - for (int row = 0; row <= columns.Length; row++) - { - string? rowCatalog = row < columns.Length ? nativeColumns.String("TABLE_CAT", row) : null; - bool matches = row < columns.Length && (rowCatalog == null || - string.Equals(rowCatalog, sourceCatalog, StringComparison.OrdinalIgnoreCase)); - if (matches && start < 0) start = row; - if (!matches && start >= 0) - { - results.Add((columns.Slice(start, row - start), true, sourceCatalog)); - start = -1; - } - } - } + result = await ExecuteMetadataCommandAsync( + sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); } - } - return results; - } - - private static bool MatchesCatalogPattern(string? pattern, string catalog) - { - if (pattern == null) return true; - var regex = new StringBuilder("^"); - bool escaped = false; - foreach (char character in pattern) - { - if (!escaped && character == '\\') + catch { - escaped = true; + // Skip catalogs we can't access (permission errors) continue; } - regex.Append(!escaped && character == '%' ? ".*" - : !escaped && character == '_' ? "." - : Regex.Escape(character.ToString())); - escaped = false; + results.Add(new MetadataBatches(result.Batches, result.IsNative, + sourceCatalog, requireExactCatalog: true)); } - if (escaped) regex.Append(Regex.Escape("\\")); - regex.Append('$'); - return Regex.IsMatch(catalog, regex.ToString(), RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, - TimeSpan.FromSeconds(1)); + return new ColumnMetadataResult(results); } private async Task> ResolveGetObjectsCatalogsAsync( diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 9969dcc2..4b0fd5ea 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1101,7 +1101,7 @@ private static async Task>> FetchAllChunkSegmentsAsync /// catalog for SHOW commands when not querying all catalogs). /// // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, the - // ExecuteNativeShowColumnsAsync catalog fanout can be removed. + // ReadColumnsAsync catalog fanout can be removed. private string? EffectiveCatalog { get @@ -1293,27 +1293,19 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT // matching Thrift behavior (Thrift RPC has no catalog filter for GetCatalogs). string sql = new ShowCatalogsCommand(null).Build(); activity?.SetTag("sql_query", sql); - var (batches, isNative) = await _connection.ExecuteNativeMetadataSqlAsync( + var result = await _connection.ExecuteMetadataCommandAsync( sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); - IsNativeMetadataResult = isNative; - if (isNative) + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) return NativeMetadataResultBuilder.Build( - batches, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); + result.Batches, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); var tableCatBuilder = new StringArray.Builder(); int count = 0; - foreach (var batch in batches) + foreach (string catalog in MetadataRowReader.Catalogs(result)) { - var catalogArray = TryGetColumn(batch, "catalog"); - if (catalogArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (!catalogArray.IsNull(i)) - { - tableCatBuilder.Append(catalogArray.GetString(i)); - count++; - } - } + tableCatBuilder.Append(catalog); + count++; } activity?.SetTag("result_count", count); @@ -1346,16 +1338,15 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo // path and the JDBC reference driver (isObjectNotFoundException). Restores // the #388 behavior; the "make SEA throw" premise was disproven — Thrift // returns empty on object-not-found too. - List batches; + MetadataBatches result; try { - var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + result = await _connection.ExecuteMetadataCommandAsync( sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); - batches = nativeResult.Batches; - IsNativeMetadataResult = nativeResult.IsNative; - if (nativeResult.IsNative) + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) return NativeMetadataResultBuilder.Build( - batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, + result.Batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, requestedCatalog: catalog, requireExactCatalog: _escapePatternWildcards); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) @@ -1366,39 +1357,14 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo return MetadataSchemaFactory.CreateEmptySchemasResult(); } - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showAllCatalogs = catalog == null; - var tableSchemaBuilder = new StringArray.Builder(); var tableCatalogBuilder = new StringArray.Builder(); int count = 0; - foreach (var batch in batches) + foreach (var row in MetadataRowReader.Schemas(result, catalog)) { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (showAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - tableSchemaBuilder.Append(schemaArray.GetString(i)); - string catalogValue = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalog ?? ""; - tableCatalogBuilder.Append(catalogValue); - count++; - } + tableSchemaBuilder.Append(row.Schema); + tableCatalogBuilder.Append(row.Catalog); + count++; } activity?.SetTag("result_count", count); @@ -1437,18 +1403,17 @@ private async Task GetTablesAsync(CancellationToken cancellationTok EscapePatternWildcardsInName(_metadataTableName)).Build(); activity?.SetTag("sql_query", sql); - List batches; + MetadataBatches result; try { - var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + result = await _connection.ExecuteMetadataCommandAsync( sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); - batches = nativeResult.Batches; - IsNativeMetadataResult = nativeResult.IsNative; - if (nativeResult.IsNative) + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) { var requestedTypes = _metadataTableTypes?.Split(',').Select(type => type.Trim()).ToArray(); return NativeMetadataResultBuilder.Build( - batches, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, + result.Batches, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, requestedCatalog: catalog, tableTypes: requestedTypes); } } @@ -1482,33 +1447,19 @@ private async Task GetTablesAsync(CancellationToken cancellationTok : null; int count = 0; - foreach (var batch in batches) + foreach (var row in MetadataRowReader.Tables(result, catalog, tableTypeFilter)) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var tableTypeArray = TryGetColumn(batch, "tableType"); - var remarksArray = TryGetColumn(batch, "remarks"); - if (catalogArray == null || schemaArray == null || tableArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; - string tableType = NativeMetadataResultBuilder.DefaultTableType( - tableTypeArray != null && !tableTypeArray.IsNull(i) ? tableTypeArray.GetString(i) : null); - if (tableTypeFilter != null && !tableTypeFilter.Contains(tableType)) continue; - tableCatBuilder.Append(catalogArray.GetString(i)); - tableSchemaBuilder.Append(schemaArray.GetString(i)); - tableNameBuilder.Append(tableArray.GetString(i)); - tableTypeBuilder.Append(tableType); - remarksBuilder.Append(remarksArray != null && !remarksArray.IsNull(i) ? remarksArray.GetString(i) : ""); - typeCatBuilder.AppendNull(); - typeSchemaBuilder.AppendNull(); - typeNameBuilder.AppendNull(); - selfRefColBuilder.AppendNull(); - refGenBuilder.AppendNull(); - count++; - } + tableCatBuilder.Append(row.Catalog); + tableSchemaBuilder.Append(row.Schema); + tableNameBuilder.Append(row.Table); + tableTypeBuilder.Append(row.TableType); + remarksBuilder.Append(row.Remarks); + typeCatBuilder.AppendNull(); + typeSchemaBuilder.AppendNull(); + typeNameBuilder.AppendNull(); + selfRefColBuilder.AppendNull(); + refGenBuilder.AppendNull(); + count++; } activity?.SetTag("result_count", count); @@ -1542,15 +1493,15 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - List<(RecordBatch Batch, bool IsNative, string? Catalog)> columnBatches; + ColumnMetadataResult columns; try { - columnBatches = await _connection.ExecuteNativeShowColumnsAsync( + columns = await _connection.ReadColumnsAsync( catalog, EscapePatternWildcardsInName(_metadataSchemaName), EscapePatternWildcardsInName(_metadataTableName), EscapePatternWildcardsInName(_metadataColumnName), - cancellationToken).ConfigureAwait(false); + cancellationToken, requireExactCatalog: _escapePatternWildcards).ConfigureAwait(false); } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { @@ -1561,71 +1512,29 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo System.Array.Empty<(string, string, string, TableInfo)>()); } - IsNativeMetadataResult = columnBatches.Count > 0 && columnBatches.All(result => result.IsNative); + IsNativeMetadataResult = columns.IsNative; if (IsNativeMetadataResult) { return NativeMetadataResultBuilder.Build( - columnBatches.Select(result => result.Batch).ToList(), + columns.Batches, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, requestedCatalog: catalog, - sourceCatalogs: columnBatches.Select(result => result.Catalog).ToList(), - requireExactCatalog: _escapePatternWildcards); + sourceCatalogs: columns.SourceCatalogs, + requireExactCatalog: _escapePatternWildcards, + requireExactSourceCatalog: catalog == null); } var tableInfos = new Dictionary(); - foreach (var (batch, isNative, sourceCatalog) in columnBatches) + foreach (var column in columns.Rows) { - var native = isNative ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns) : null; - var catalogArray = isNative ? null : TryGetColumn(batch, "catalogName"); - var schemaArray = isNative ? null : TryGetColumn(batch, "namespace"); - var tableArray = isNative ? null : TryGetColumn(batch, "tableName"); - var colNameArray = isNative ? null : TryGetColumn(batch, "col_name"); - var columnTypeArray = isNative ? null : TryGetColumn(batch, "columnType"); - var isNullableArray = isNative ? null : TryGetColumn(batch, "isNullable"); - - if (!isNative && (catalogArray == null || schemaArray == null || tableArray == null || - colNameArray == null || columnTypeArray == null)) continue; - - for (int i = 0; i < batch.Length; i++) - { - string? cat = isNative ? native!.String("TABLE_CAT", i) ?? sourceCatalog ?? catalog - : catalogArray!.IsNull(i) ? null : catalogArray.GetString(i); - string? sch = isNative ? native!.String("TABLE_SCHEM", i) - : schemaArray!.IsNull(i) ? null : schemaArray.GetString(i); - string? tbl = isNative ? native!.String("TABLE_NAME", i) - : tableArray!.IsNull(i) ? null : tableArray.GetString(i); - string? colName = isNative ? native!.String("COLUMN_NAME", i) - : colNameArray!.IsNull(i) ? null : colNameArray.GetString(i); - string? colType = isNative ? native!.String("TYPE_NAME", i) - : columnTypeArray!.IsNull(i) ? null : columnTypeArray.GetString(i); - if (cat == null || sch == null || tbl == null || colName == null || colType == null) continue; - if (isNative && _escapePatternWildcards && catalog != null && - !string.Equals(catalog, cat, StringComparison.OrdinalIgnoreCase)) continue; - - string key = $"{cat}.{sch}.{tbl}"; - - if (!tableInfos.ContainsKey(key)) - tableInfos[key] = (cat, sch, tbl, new TableInfo("TABLE")); - - var entry = tableInfos[key]; - bool nullable = isNative ? native!.Integer("NULLABLE", i) == 1 - : isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - int position = isNative - ? checked((int)(native!.Integer("ORDINAL_POSITION", i) ?? 0)) - : entry.info.ColumnName.Count; - - ColumnMetadataHelper.PopulateTableInfoFromTypeName( - entry.info, - colName, - colType, - position, - nullable, - columnDefault: isNative ? native!.String("COLUMN_DEF", i) : null, - isAutoIncrement: isNative && string.Equals(native!.String("IS_AUTO_INCREMENT", i), "YES", StringComparison.OrdinalIgnoreCase)); - } + if (column.Catalog == null || column.Schema == null || column.Table == null) continue; + string key = $"{column.Catalog}.{column.Schema}.{column.Table}"; + if (!tableInfos.ContainsKey(key)) + tableInfos[key] = (column.Catalog, column.Schema, column.Table, new TableInfo("TABLE")); + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfos[key].info, column.Name, column.TypeName, column.Ordinal, column.Nullable, + columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); } activity?.SetTag("result_tables", tableInfos.Count); @@ -1869,11 +1778,11 @@ private async Task GetPrimaryKeysAsyncNoThrow( List batches; try { - var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + var metadataResult = await _connection.ExecuteMetadataCommandAsync( sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); - batches = nativeResult.Batches; - IsNativeMetadataResult = nativeResult.IsNative; - if (nativeResult.IsNative) + batches = metadataResult.Batches; + IsNativeMetadataResult = metadataResult.IsNative; + if (metadataResult.IsNative) { return NativeMetadataResultBuilder.Build( batches, MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys); @@ -1999,11 +1908,11 @@ private async Task GetCrossReferenceAsyncNoThrow( List batches; try { - var nativeResult = await _connection.ExecuteNativeMetadataSqlAsync( + var metadataResult = await _connection.ExecuteMetadataCommandAsync( sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); - batches = nativeResult.Batches; - IsNativeMetadataResult = nativeResult.IsNative; - if (nativeResult.IsNative) + batches = metadataResult.Batches; + IsNativeMetadataResult = metadataResult.IsNative; + if (metadataResult.IsNative) { return NativeMetadataResultBuilder.Build( batches, MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs new file mode 100644 index 00000000..760a1ca2 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Types; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class MetadataRowReaderTests + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Catalogs_ApplyTheSamePatternForBothFormats(bool native) + { + using var batch = Strings((native ? "TABLE_CAT" : "catalog", + new string?[] { "ma_catalog", "maxcatalog", "other" })); + var result = new MetadataBatches(new List { batch }, native); + + Assert.Equal(new[] { "ma_catalog" }, MetadataRowReader.Catalogs(result, @"ma\_%")); + } + + [Fact] + public void ShowColumns_OrdinalsContinueAcrossBatchesPerTable() + { + using var first = ShowColumns(("t", "a"), ("other", "b"), ("t", "c")); + using var second = ShowColumns(("t", "d")); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { first, second }, false), + }); + + Assert.Equal(new[] { 0, 0, 1, 2 }, result.Rows.Select(row => row.Ordinal)); + Assert.All(result.Rows, row => Assert.True(row.Nullable)); + Assert.False(result.IsNative); + } + + [Fact] + public void ShowColumns_PartialIdentifiersStillProvideTableSchemaFields() + { + using var batch = Strings( + ("col_name", new string?[] { "a" }), + ("columnType", new string?[] { "INT" }), + ("isNullable", new string?[] { "false" })); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { batch }, false), + }); + + var row = Assert.Single(result.Rows); + Assert.Null(row.Catalog); + Assert.Null(row.Schema); + Assert.Null(row.Table); + Assert.Equal("a", row.Name); + Assert.Equal("INT", row.TypeName); + Assert.False(row.Nullable); + } + + [Fact] + public void NativeColumns_FilterExactCatalogAndFillMissingCatalog() + { + using var batch = NativeColumns("foo_bar", "fooxbar", null); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { batch }, true, "foo_bar", requireExactCatalog: true), + }); + + Assert.True(result.IsNative); + Assert.Equal(3, Assert.Single(result.Batches).Length); + Assert.Equal(2, result.Rows.Count); + Assert.All(result.Rows, row => + { + Assert.Equal("foo_bar", row.Catalog); + Assert.Equal(7, row.Ordinal); + Assert.Equal("7", row.Default); + Assert.True(row.Nullable); + Assert.True(row.IsAutoIncrement); + }); + } + + [Fact] + public void MixedColumns_PreserveServerOrdinalsAndCountShowRows() + { + using var native = NativeColumns("main"); + using var show = ShowColumns(("t", "b")); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { native }, true, "main"), + new MetadataBatches(new List { show }, false, "main"), + }); + + Assert.False(result.IsNative); + Assert.Equal(new[] { 7, 1 }, result.Rows.Select(row => row.Ordinal)); + Assert.Equal(new[] { "a", "b" }, result.Rows.Select(row => row.Name)); + } + + private static RecordBatch ShowColumns(params (string Table, string Name)[] rows) + => Strings( + ("catalogName", rows.Select(_ => (string?)"main").ToArray()), + ("namespace", rows.Select(_ => (string?)"default").ToArray()), + ("tableName", rows.Select(row => (string?)row.Table).ToArray()), + ("col_name", rows.Select(row => (string?)row.Name).ToArray()), + ("columnType", rows.Select(_ => (string?)"INT").ToArray())); + + private static RecordBatch Strings(params (string Name, string?[] Values)[] columns) + { + var schema = new Schema(columns.Select(column => + new Field(column.Name, StringType.Default, true)), null); + var arrays = columns.Select(column => + { + var builder = new StringArray.Builder(); + foreach (string? value in column.Values) + { + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + }).ToArray(); + return new RecordBatch(schema, arrays, columns[0].Values.Length); + } + + private static RecordBatch NativeColumns(params string?[] catalogs) + { + var schema = new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23), null); + var arrays = schema.FieldsList.Select(field => + { + if (field.DataType.TypeId == ArrowTypeId.String) + { + var builder = new StringArray.Builder(); + foreach (string? catalog in catalogs) + { + string? value = field.Name switch + { + "TABLE_CAT" => catalog, + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "COLUMN_NAME" => "a", + "TYPE_NAME" => "INT", + "COLUMN_DEF" => "7", + "IS_AUTO_INCREMENT" => "YES", + _ => null, + }; + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + } + int number = field.Name == "ORDINAL_POSITION" ? 7 : field.Name == "NULLABLE" ? 1 : 0; + return field.DataType.TypeId == ArrowTypeId.Int16 + ? (IArrowArray)new Int16Array.Builder().AppendRange( + Enumerable.Repeat((short)number, catalogs.Length)).Build() + : new Int32Array.Builder().AppendRange(Enumerable.Repeat(number, catalogs.Length)).Build(); + }).ToArray(); + return new RecordBatch(schema, arrays, catalogs.Length); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs index b793e04a..07a6c2dd 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs @@ -58,7 +58,7 @@ public class StatementExecutionGetObjectsCatalogPatternTests (@"foo\_bar", new[] { "foo_bar" }), ("foo_bar", new[] { "foo_bar", "fooxbar" }), }; - foreach (int mode in new[] { 0, 1, 2 }) + foreach (int mode in new[] { 0, 1, 2, 3 }) { foreach (AdbcConnection.GetObjectsDepth depth in new[] { AdbcConnection.GetObjectsDepth.Tables, AdbcConnection.GetObjectsDepth.All }) @@ -145,6 +145,32 @@ public void MetadataCommand_ResetsNativeFlagForSyntheticResults() Assert.False(statement.IsNativeMetadataResult); } + [Fact] + public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdinals() + { + using HttpClient http = CreateHttpClient(3, new List()); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SqlQuery = "GetColumns"; + using IArrowArrayStream stream = statement.ExecuteQuery().Stream!; + using RecordBatch batch = (await stream.ReadNextRecordBatchAsync())!; + + Assert.False(statement.IsNativeMetadataResult); + Assert.Equal(s_catalogs.Length, batch.Length); + var catalogs = (StringArray)batch.Column("TABLE_CAT"); + var ordinals = (Int32Array)batch.Column("ORDINAL_POSITION"); + var defaults = (StringArray)batch.Column("COLUMN_DEF"); + var autoIncrement = (StringArray)batch.Column("IS_AUTO_INCREMENT"); + for (int row = 0; row < batch.Length; row++) + { + Assert.Equal(0, ordinals.GetValue(row)); + bool native = catalogs.GetString(row) == "main"; + Assert.Equal(native ? "7" : null, defaults.GetString(row)); + Assert.Equal(native ? "YES" : "NO", autoIncrement.GetString(row)); + } + } + private static StatementExecutionConnection CreateConnection(HttpClient http) { Dictionary properties = new Dictionary @@ -174,9 +200,10 @@ private static HttpClient CreateHttpClient(int mode, List statements) string sql = json.RootElement.GetProperty("statement").GetString()!; statements.Add(sql); string operation = request.Headers.GetValues("x-databricks-metadata-operation-type").Single(); - bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables"); string? requestedCatalog = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs" ? null : s_catalogs.Single(catalog => sql.Contains($"`{catalog}`")); + bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || + (mode == 3 && (operation != "GetColumns" || requestedCatalog == "main")); string[] catalogs = requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; if (native && requestedCatalog != null && operation is "GetTables" or "GetColumns") catalogs = catalogs.Concat(new[] { requestedCatalog == "other" ? "main" : "other" }).ToArray(); @@ -258,6 +285,8 @@ private static RecordBatch CreateBatch(string operation, bool native, string[] c "TABLE_TYPE" or "tableType" => "TABLE", "COLUMN_NAME" or "col_name" => "a", "TYPE_NAME" or "columnType" => "INT", + "COLUMN_DEF" => "7", + "IS_AUTO_INCREMENT" => "YES", "isNullable" => "true", _ => null, }; diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 7a3972e7..33608393 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -201,7 +201,7 @@ public async Task MetadataConnection_AlwaysRequestsNativeMetadata() }; using var connection = new StatementExecutionConnection(properties, http); - await connection.ExecuteNativeMetadataSqlAsync("SHOW CATALOGS", MetadataOperation.GetCatalogs, CancellationToken.None); + await connection.ExecuteMetadataCommandAsync("SHOW CATALOGS", MetadataOperation.GetCatalogs, CancellationToken.None); Assert.True(hasSyncHeader); Assert.True(hasOperationHeader); diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 93745360..cd3e4171 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -193,11 +193,10 @@ public async Task ColumnFanout_KeepsTheQueriedCatalog() using var http = CreateHttpClient(); using var connection = CreateConnection(http); - var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); + var result = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); - Assert.Equal(2, batches.Count); - Assert.Equal("main", batches[0].Catalog); - Assert.Equal("other", batches[1].Catalog); + Assert.Equal(2, result.Batches.Count); + Assert.Equal(new[] { "main", "other" }, result.SourceCatalogs); } [Theory] @@ -224,7 +223,7 @@ public async Task ColumnFanout_MalformedNativeColumns_PropagatesValidationError( using var connection = CreateConnection(http); var exception = await Assert.ThrowsAsync(() => - connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None)); + connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None)); Assert.StartsWith("Invalid native GetColumns result:", exception.Message); Assert.Contains(expectedError, exception.Message); @@ -236,21 +235,17 @@ public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() using var http = CreateHttpClient(overlappingNativeColumns: true); using var connection = CreateConnection(http); - var batches = await connection.ExecuteNativeShowColumnsAsync(null, null, null, null, CancellationToken.None); + var columns = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); - Assert.Equal(2, batches.Count); - Assert.All(batches, result => - { - Assert.True(result.IsNative); - Assert.Equal(1, result.Batch.Length); - Assert.Equal(result.Catalog, ((StringArray)result.Batch.Column("TABLE_CAT")).GetString(0)); - }); - Assert.Equal(new[] { "foo_bar", "fooxbar" }, batches.Select(result => result.Catalog)); + Assert.True(columns.IsNative); + Assert.Equal(2, columns.Batches.Count); + Assert.All(columns.Batches, batch => Assert.Equal(2, batch.Length)); + Assert.Equal(new[] { "foo_bar", "fooxbar" }, columns.Rows.Select(row => row.Catalog)); var result = NativeMetadataResultBuilder.Build( - batches.Select(batch => batch.Batch).ToArray(), + columns.Batches, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, - sourceCatalogs: batches.Select(batch => batch.Catalog).ToArray()); + sourceCatalogs: columns.SourceCatalogs, requireExactSourceCatalog: true); using var stream = result.Stream!; Assert.Equal(2, result.RowCount); From e8d1d36d60fd200f2320a4380e73cf52855ed9e7 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 04:34:27 +0000 Subject: [PATCH 15/22] fix(csharp): share native metadata rules and enforce CI coverage Signed-off-by: Vu Anh Phung --- .github/workflows/e2e-tests.yml | 1 + csharp/README.md | 6 +- .../StatementExecution/MetadataRowReader.cs | 11 +++- .../NativeMetadataResultBuilder.cs | 34 ++++------- .../StatementExecution/SeaMetadataE2ETests.cs | 60 ++++++++++++++++++- csharp/test/MetadataRecordingConnection.cs | 57 ++++++++++++++++++ .../MetadataRowReaderTests.cs | 13 +++- .../NativeMetadataResultBuilderTests.cs | 57 ++++++++++++++++++ ...tExecutionGetObjectsCatalogPatternTests.cs | 48 ++++++++++++++- 9 files changed, 253 insertions(+), 34 deletions(-) create mode 100644 csharp/test/MetadataRecordingConnection.cs diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 22609b02..75b4c272 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -230,6 +230,7 @@ jobs: "query": "SELECT * FROM main.${PER_RUN_SCHEMA}.adbc_testing_table", "expectedResults": 12, "isCITesting": true, + "requireNativeMetadata": ${{ matrix.protocol == 'rest' }}, "tracePropagationEnabled": "true", "traceParentHeaderName": "traceparent", "traceStateEnabled": "false", diff --git a/csharp/README.md b/csharp/README.md index 664b660e..02d288af 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -613,8 +613,10 @@ dotnet test --filter "FullyQualifiedName~CloudFetch" ``` For a native-capable warehouse, set `"requireNativeMetadata": true` in the test -configuration and run `dotnet test --filter "FullyQualifiedName~NativeMetadata_UsesNativeResponse"`. -These tests fail if any supported metadata operation falls back to SHOW. +configuration and run `dotnet test --filter "FullyQualifiedName~SeaMetadataE2ETests.NativeMetadata_"`. +These tests cover the six flat operations, `GetObjects`, and `GetTableSchema`, and +fail if any metadata request falls back to SHOW. The REST E2E CI job enables this +requirement; local configurations can leave it off for fallback-capable warehouses. #### Verbose Test Output diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs index 50ad0660..423b8b71 100644 --- a/csharp/src/StatementExecution/MetadataRowReader.cs +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -268,8 +268,17 @@ private static IEnumerable ShowColumns(RecordBatch batch, Dictionary< } internal static int CompareTables(TableRow left, TableRow right) + => CompareTables( + (left.Catalog, left.Schema, left.Table, left.TableType), + (right.Catalog, right.Schema, right.Table, right.TableType)); + + internal static int CompareTables( + (string? Catalog, string? Schema, string? Table, string? TableType) left, + (string? Catalog, string? Schema, string? Table, string? TableType) right) { - int order = StringComparer.Ordinal.Compare(left.TableType, right.TableType); + int order = StringComparer.Ordinal.Compare( + NativeMetadataResultBuilder.DefaultTableType(left.TableType), + NativeMetadataResultBuilder.DefaultTableType(right.TableType)); if (order != 0) return order; order = StringComparer.Ordinal.Compare(left.Catalog, right.Catalog); if (order != 0) return order; diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 4958d185..56fd171b 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -59,7 +59,7 @@ internal static QueryResult Build( // Native metadata can expand wildcard characters in a quoted catalog. if (catalogField != null && requestedCatalog != null && (operation == MetadataOperation.GetTables || requireExactCatalog) && - !string.Equals(requestedCatalog, columns.String(catalogField, row) ?? sourceCatalog, StringComparison.OrdinalIgnoreCase)) + !MetadataRowReader.MatchesCatalog(requestedCatalog, columns.String(catalogField, row) ?? sourceCatalog)) continue; if (operation == MetadataOperation.GetTables && tableTypes != null && !tableTypes.Contains(DefaultTableType(columns.String("TABLE_TYPE", row)))) @@ -77,29 +77,15 @@ internal static QueryResult Build( if (operation == MetadataOperation.GetTables) { - int[] sortColumns = { 3, 0, 1, 2 }; - rows.Sort((left, right) => - { - foreach (int column in sortColumns) - { - string? leftValue = left.Columns.String(schema.FieldsList[column].Name, left.Index); - string? rightValue = right.Columns.String(schema.FieldsList[column].Name, right.Index); - if (column == 3) - { - leftValue = DefaultTableType(leftValue); - rightValue = DefaultTableType(rightValue); - } - if (column == 0) - { - leftValue ??= left.Catalog; - rightValue ??= right.Catalog; - } - int result = StringComparer.Ordinal.Compare( - leftValue, rightValue); - if (result != 0) return result; - } - return 0; - }); + rows.Sort((left, right) => MetadataRowReader.CompareTables( + (left.Columns.String("TABLE_CAT", left.Index) ?? left.Catalog, + left.Columns.String("TABLE_SCHEM", left.Index), + left.Columns.String("TABLE_NAME", left.Index), + left.Columns.String("TABLE_TYPE", left.Index)), + (right.Columns.String("TABLE_CAT", right.Index) ?? right.Catalog, + right.Columns.String("TABLE_SCHEM", right.Index), + right.Columns.String("TABLE_NAME", right.Index), + right.Columns.String("TABLE_TYPE", right.Index)))); } var arrays = new List(schema.FieldsList.Count); diff --git a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs index 13446a6d..f63f47f8 100644 --- a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs +++ b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs @@ -23,6 +23,7 @@ using Apache.Arrow; using Apache.Arrow.Adbc; using Apache.Arrow.Adbc.Tests; +using Apache.Arrow.Adbc.Tests.Metadata; using AdbcDrivers.HiveServer2; using Xunit; using Xunit.Abstractions; @@ -64,6 +65,32 @@ private void SkipIfNotConfigured() Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable), "Test configuration not available"); } + private void SkipIfNativeMetadataNotRequired() + { + SkipIfNotConfigured(); + Skip.IfNot(TestConfiguration.RequireNativeMetadata, + "Set requireNativeMetadata=true in the test configuration for a native-capable warehouse."); + } + + private async Task CreateMetadataRecordingConnection() + { + var parameters = GetDriverParameters(TestConfiguration); + parameters[DatabricksParameters.Protocol] = "rest"; + parameters[DatabricksParameters.EnableMultipleCatalogSupport] = "true"; + var connection = new MetadataRecordingConnection(parameters); + try + { + await connection.OpenAsync(); + connection.Statements.Clear(); + return connection; + } + catch + { + connection.Dispose(); + throw; + } + } + // Connection on whatever protocol the test suite was configured with (driver // parameters / config file). Tests never pick the protocol themselves — the run // argument decides it, so the SEA/Reyden nightly runs them over REST and the @@ -223,9 +250,7 @@ public async Task GetTables_TypesFilter_IsCaseSensitive() public async Task NativeMetadata_UsesNativeResponse( string command, string table, string? foreignTable, string expectedColumn, string expectedValue) { - SkipIfNotConfigured(); - Skip.IfNot(TestConfiguration.RequireNativeMetadata, - "Set requireNativeMetadata=true in the test configuration for a native-capable warehouse."); + SkipIfNativeMetadataNotRequired(); using var conn = CreateConnection(new Dictionary(RestProtocol) { @@ -238,6 +263,35 @@ public async Task NativeMetadata_UsesNativeResponse( Assert.Contains(rows, row => row[expectedColumn] == expectedValue); } + [SkippableFact] + public async Task NativeMetadata_GetObjects_UsesNativeResponse() + { + SkipIfNativeMetadataNotRequired(); + using var connection = await CreateMetadataRecordingConnection(); + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, TestCatalog, TestSchema, TestTable, null, null); + using var batch = await stream.ReadNextRecordBatchAsync(); + Assert.NotNull(batch); + + var catalog = Assert.Single(GetObjectsParser.ParseCatalog(batch, null), + catalog => catalog.Name == TestCatalog); + var schema = Assert.Single(catalog.DbSchemas!, schema => schema.Name == TestSchema); + var table = Assert.Single(schema.Tables!, table => table.Name == TestTable); + Assert.Contains(table.Columns!, column => column.Name == "c_int"); + connection.AssertNativeResponses("SHOW CATALOGS", "SHOW SCHEMAS", "SHOW TABLES", "SHOW COLUMNS"); + } + + [SkippableFact] + public async Task NativeMetadata_GetTableSchema_UsesNativeResponse() + { + SkipIfNativeMetadataNotRequired(); + using var connection = await CreateMetadataRecordingConnection(); + var schema = connection.GetTableSchema(TestCatalog, TestSchema, "cross_ref_customers"); + + Assert.Equal("customer_id", schema.FieldsList[0].Name); + connection.AssertNativeResponses("SHOW COLUMNS"); + } + [SkippableFact] public async Task GetColumns_CatalogMatchAll_WithEscaping_ReturnsEmpty() { diff --git a/csharp/test/MetadataRecordingConnection.cs b/csharp/test/MetadataRecordingConnection.cs new file mode 100644 index 00000000..9f023366 --- /dev/null +++ b/csharp/test/MetadataRecordingConnection.cs @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Net.Http; +using AdbcDrivers.Databricks.StatementExecution; +using Apache.Arrow.Adbc; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests +{ + internal sealed class MetadataRecordingConnection : StatementExecutionConnection + { + internal MetadataRecordingConnection(IReadOnlyDictionary properties) + : base(properties) + { + } + + internal MetadataRecordingConnection(IReadOnlyDictionary properties, HttpClient http) + : base(properties, http) + { + } + + internal List Statements { get; } = new(); + + public override AdbcStatement CreateStatement() + { + var statement = (StatementExecutionStatement)base.CreateStatement(); + Statements.Add(statement); + return statement; + } + + internal void AssertNativeResponses(params string[] commands) + { + Assert.NotEmpty(Statements); + foreach (string command in commands) + Assert.Contains(Statements, statement => + statement.SqlQuery?.StartsWith(command, StringComparison.Ordinal) == true); + Assert.All(Statements, statement => Assert.True(statement.IsNativeMetadataResult, + $"{statement.SqlQuery} fell back to SHOW; expected is_native_metadata_result: true.")); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs index 760a1ca2..0eaf7a09 100644 --- a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Linq; +using System.Threading.Tasks; using AdbcDrivers.Databricks.StatementExecution; using AdbcDrivers.HiveServer2.Hive2; using Apache.Arrow; @@ -75,7 +76,7 @@ public void ShowColumns_PartialIdentifiersStillProvideTableSchemaFields() } [Fact] - public void NativeColumns_FilterExactCatalogAndFillMissingCatalog() + public async Task NativeColumns_FilterExactCatalogAndFillMissingCatalog() { using var batch = NativeColumns("foo_bar", "fooxbar", null); var result = new ColumnMetadataResult(new[] @@ -94,6 +95,16 @@ public void NativeColumns_FilterExactCatalogAndFillMissingCatalog() Assert.True(row.Nullable); Assert.True(row.IsAutoIncrement); }); + + var flat = NativeMetadataResultBuilder.Build(result.Batches, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, + sourceCatalogs: result.SourceCatalogs, requireExactSourceCatalog: true); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(result.Rows.Count, output.Length); + var catalogs = (StringArray)output.Column("TABLE_CAT"); + Assert.Equal(result.Rows.Select(row => row.Catalog), + Enumerable.Range(0, output.Length).Select(row => catalogs.GetString(row))); } [Fact] diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index 2ce5c130..8ee73517 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -93,6 +93,63 @@ public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCat Assert.Equal("TABLE", ((StringArray)batch.Column(3)).GetString(0)); } + [Theory] + [InlineData(null, false)] + [InlineData(null, true)] + [InlineData("main", false)] + [InlineData("main", true)] + public async Task GetTables_FlatAndDecodedResultsShareCatalogFilteringAndOrdering( + string? catalog, bool filterTypes) + { + var schema = MetadataSchemaFactory.CreateTablesSchema(); + (string? Catalog, string Schema, string Table, string? Type)[] tables = + { + ("other", "a", "other_table", "TABLE"), + (null, "b", "b_a", ""), + ("main", "a", "a_view", "VIEW"), + ("main", "a", "a_z", "TABLE"), + ("main", "a", "a_a", null), + ("main", "b", "b_z", "TABLE"), + ("main", "a", "a_b", "TABLE"), + }; + var arrays = schema.FieldsList.Select(field => + { + var builder = new StringArray.Builder(); + foreach (var table in tables) + { + string? value = field.Name switch + { + "TABLE_CAT" => table.Catalog, + "TABLE_SCHEM" => table.Schema, + "TABLE_NAME" => table.Table, + "TABLE_TYPE" => table.Type, + _ => "server_value", + }; + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, tables.Length); + string[]? tableTypes = filterTypes ? new[] { "TABLE" } : null; + var decoded = MetadataRowReader.Tables( + new MetadataBatches(new List { nativeBatch }, true), catalog, tableTypes); + var result = NativeMetadataResultBuilder.Build( + new[] { nativeBatch }, schema, MetadataOperation.GetTables, + requestedCatalog: catalog, tableTypes: tableTypes); + using var reader = result.Stream!; + using var batch = (await reader.ReadNextRecordBatchAsync())!; + var names = (StringArray)batch.Column("TABLE_NAME"); + string[] expected = catalog == null + ? new[] { "b_a", "a_a", "a_b", "a_z", "b_z", "other_table" } + : new[] { "a_a", "a_b", "a_z", "b_a", "b_z" }; + if (!filterTypes) expected = expected.Concat(new[] { "a_view" }).ToArray(); + + Assert.Equal(expected, decoded.Select(row => row.Table)); + Assert.Equal(expected, Enumerable.Range(0, batch.Length).Select(row => names.GetString(row))); + Assert.All(Enumerable.Range(0, batch.Length), row => + Assert.Equal("server_value", ((StringArray)batch.Column("TYPE_CAT")).GetString(row))); + } + [Theory] [InlineData((int)MetadataOperation.GetSchemas, "%", true, 0)] [InlineData((int)MetadataOperation.GetColumns, "compar%", true, 0)] diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs index 07a6c2dd..ab91d4db 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs @@ -82,7 +82,8 @@ public async Task GetObjects_ResolvesCatalogPatterns( Assert.NotNull(batch); List catalogs = GetObjectsParser.ParseCatalog(batch, null); - Assert.Equal(expectedCatalogs.OrderBy(name => name), catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); + Assert.Equal(expectedCatalogs.OrderBy(name => name), + catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); foreach (AdbcCatalog catalog in catalogs) { AdbcDbSchema schema = Assert.Single(catalog.DbSchemas!); @@ -171,7 +172,46 @@ public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdin } } - private static StatementExecutionConnection CreateConnection(HttpClient http) + [Theory] + [InlineData(0, false, false)] + [InlineData(1, false, true)] + [InlineData(2, false, false)] + [InlineData(3, false, true)] + [InlineData(0, true, false)] + [InlineData(1, true, true)] + [InlineData(2, true, false)] + [InlineData(3, true, false)] + public void NativeResponseAssertions_CheckInternalMetadataStatements( + int mode, bool getObjects, bool expectedNative) + { + using HttpClient http = CreateHttpClient(mode, new List()); + using var connection = (MetadataRecordingConnection)CreateConnection(http, recordMetadata: true); + string[] commands; + if (getObjects) + { + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, "ma%", "default", "t1", null, null); + commands = new[] { "SHOW CATALOGS", "SHOW SCHEMAS", "SHOW TABLES", "SHOW COLUMNS" }; + } + else + { + Assert.NotEmpty(connection.GetTableSchema("main", "default", "t1").FieldsList); + commands = new[] { "SHOW COLUMNS" }; + } + + if (expectedNative) + { + connection.AssertNativeResponses(commands); + } + else + { + var exception = Assert.ThrowsAny( + () => connection.AssertNativeResponses(commands)); + Assert.Contains("fell back to SHOW", exception.Message); + } + } + + private static StatementExecutionConnection CreateConnection(HttpClient http, bool recordMetadata = false) { Dictionary properties = new Dictionary { @@ -179,7 +219,9 @@ private static StatementExecutionConnection CreateConnection(HttpClient http) [DatabricksParameters.WarehouseId] = "wh-1", [SparkParameters.AccessToken] = "token", }; - return new StatementExecutionConnection(properties, http); + return recordMetadata + ? new MetadataRecordingConnection(properties, http) + : new StatementExecutionConnection(properties, http); } private static HttpClient CreateHttpClient(int mode, List statements) From 95d16203e3dac4e185121cef6932e36ba881181d Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 04:44:57 +0000 Subject: [PATCH 16/22] test(csharp): always require native responses in SEA metadata E2Es Signed-off-by: Vu Anh Phung --- .github/workflows/e2e-tests.yml | 1 - csharp/README.md | 9 ++++----- csharp/test/E2E/DatabricksTestConfiguration.cs | 3 --- .../E2E/StatementExecution/SeaMetadataE2ETests.cs | 13 +++---------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 75b4c272..22609b02 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -230,7 +230,6 @@ jobs: "query": "SELECT * FROM main.${PER_RUN_SCHEMA}.adbc_testing_table", "expectedResults": 12, "isCITesting": true, - "requireNativeMetadata": ${{ matrix.protocol == 'rest' }}, "tracePropagationEnabled": "true", "traceParentHeaderName": "traceparent", "traceStateEnabled": "false", diff --git a/csharp/README.md b/csharp/README.md index 02d288af..9446c1e4 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -612,11 +612,10 @@ Run tests matching a pattern: dotnet test --filter "FullyQualifiedName~CloudFetch" ``` -For a native-capable warehouse, set `"requireNativeMetadata": true` in the test -configuration and run `dotnet test --filter "FullyQualifiedName~SeaMetadataE2ETests.NativeMetadata_"`. -These tests cover the six flat operations, `GetObjects`, and `GetTableSchema`, and -fail if any metadata request falls back to SHOW. The REST E2E CI job enables this -requirement; local configurations can leave it off for fallback-capable warehouses. +Run `dotnet test --filter "FullyQualifiedName~SeaMetadataE2ETests.NativeMetadata_"` +against a configured native-capable warehouse. These tests cover the six flat +operations, `GetObjects`, and `GetTableSchema`, and always fail if any metadata +request falls back to SHOW. #### Verbose Test Output diff --git a/csharp/test/E2E/DatabricksTestConfiguration.cs b/csharp/test/E2E/DatabricksTestConfiguration.cs index 23c57f82..537e74f0 100644 --- a/csharp/test/E2E/DatabricksTestConfiguration.cs +++ b/csharp/test/E2E/DatabricksTestConfiguration.cs @@ -78,8 +78,5 @@ public class DatabricksTestConfiguration : SparkTestConfiguration [JsonPropertyName("protocol"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] public string Protocol { get; set; } = string.Empty; - - [JsonPropertyName("requireNativeMetadata"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)] - public bool RequireNativeMetadata { get; set; } } } diff --git a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs index f63f47f8..474c50be 100644 --- a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs +++ b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs @@ -65,13 +65,6 @@ private void SkipIfNotConfigured() Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable), "Test configuration not available"); } - private void SkipIfNativeMetadataNotRequired() - { - SkipIfNotConfigured(); - Skip.IfNot(TestConfiguration.RequireNativeMetadata, - "Set requireNativeMetadata=true in the test configuration for a native-capable warehouse."); - } - private async Task CreateMetadataRecordingConnection() { var parameters = GetDriverParameters(TestConfiguration); @@ -250,7 +243,7 @@ public async Task GetTables_TypesFilter_IsCaseSensitive() public async Task NativeMetadata_UsesNativeResponse( string command, string table, string? foreignTable, string expectedColumn, string expectedValue) { - SkipIfNativeMetadataNotRequired(); + SkipIfNotConfigured(); using var conn = CreateConnection(new Dictionary(RestProtocol) { @@ -266,7 +259,7 @@ public async Task NativeMetadata_UsesNativeResponse( [SkippableFact] public async Task NativeMetadata_GetObjects_UsesNativeResponse() { - SkipIfNativeMetadataNotRequired(); + SkipIfNotConfigured(); using var connection = await CreateMetadataRecordingConnection(); using var stream = connection.GetObjects( AdbcConnection.GetObjectsDepth.All, TestCatalog, TestSchema, TestTable, null, null); @@ -284,7 +277,7 @@ public async Task NativeMetadata_GetObjects_UsesNativeResponse() [SkippableFact] public async Task NativeMetadata_GetTableSchema_UsesNativeResponse() { - SkipIfNativeMetadataNotRequired(); + SkipIfNotConfigured(); using var connection = await CreateMetadataRecordingConnection(); var schema = connection.GetTableSchema(TestCatalog, TestSchema, "cross_ref_customers"); From 8a070223ac7664948564e2489bb07ed2b601df98 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 04:53:22 +0000 Subject: [PATCH 17/22] fix(csharp): enforce exact catalog scope in GetTableSchema Signed-off-by: Vu Anh Phung --- csharp/README.md | 5 --- .../StatementExecutionConnection.cs | 3 +- .../StatementExecutionNativeCatalogTests.cs | 32 ++++++++++++++++--- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/csharp/README.md b/csharp/README.md index 9446c1e4..315866ce 100644 --- a/csharp/README.md +++ b/csharp/README.md @@ -612,11 +612,6 @@ Run tests matching a pattern: dotnet test --filter "FullyQualifiedName~CloudFetch" ``` -Run `dotnet test --filter "FullyQualifiedName~SeaMetadataE2ETests.NativeMetadata_"` -against a configured native-capable warehouse. These tests cover the six flat -operations, `GetObjects`, and `GetTableSchema`, and always fail if any metadata -request falls back to SHOW. - #### Verbose Test Output Get detailed test output: diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 68239949..ac84a3f5 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -716,7 +716,8 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string ColumnMetadataResult columns; try { - columns = ReadColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) + columns = ReadColumnsAsync( + resolvedCatalog, dbSchema, tableName, null, cts.Token, requireExactCatalog: true) .GetAwaiter().GetResult(); } catch (Exception ex) when (cts.IsCancellationRequested && ex is not TimeoutException) diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index cd3e4171..4b1be8f2 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -52,13 +52,13 @@ private static byte[] ArrowStrings(string name, params string[] values) return raw.ToArray(); } - private static byte[] ArrowNativeColumns(Field[] fields) + private static byte[] ArrowNativeColumns(Field[] fields, string nativeCatalog) { var schema = new Schema(fields, null); var arrays = fields.Select(field => { if (field.Name == "TABLE_CAT") - return (IArrowArray)new StringArray.Builder().Append("foo_bar").Append("fooxbar").Build(); + return (IArrowArray)new StringArray.Builder().Append(nativeCatalog).Append("fooxbar").Build(); string value = field.Name switch { "TABLE_SCHEM" => "default", @@ -87,15 +87,16 @@ private static byte[] ArrowNativeColumns(Field[] fields) return raw.ToArray(); } - private static HttpClient CreateHttpClient(bool overlappingNativeColumns = false, Field[]? columnFields = null) + private static HttpClient CreateHttpClient( + bool overlappingNativeColumns = false, Field[]? columnFields = null, string nativeCatalog = "foo_bar") { byte[] catalogs = overlappingNativeColumns - ? ArrowStrings("TABLE_CAT", "foo_bar", "fooxbar") + ? ArrowStrings("TABLE_CAT", nativeCatalog, "fooxbar") : ArrowStrings("TABLE_CAT", "main", "other"); Field[] nativeColumnFields = columnFields ?? MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); byte[] columns = overlappingNativeColumns - ? ArrowNativeColumns(nativeColumnFields) + ? ArrowNativeColumns(nativeColumnFields, nativeCatalog) : ArrowStrings("col_name", "a"); var handler = new Mock(); handler.Protected() @@ -229,6 +230,27 @@ public async Task ColumnFanout_MalformedNativeColumns_PropagatesValidationError( Assert.Contains(expectedError, exception.Message); } + [Theory] + [InlineData("foo_bar", "foo_bar", 1)] + [InlineData("FOO_BAR", "foo_bar", 1)] + [InlineData("foo%bar", "foo%bar", 1)] + [InlineData("FOO%BAR", "foo%bar", 1)] + [InlineData("fooxbar", "foo_bar", 1)] + [InlineData("missing", "foo_bar", 0)] + [InlineData(null, "foo_bar", 2)] + [InlineData("SPARK", "foo_bar", 2)] + public void GetTableSchema_UsesExactNativeCatalogScope( + string? requestedCatalog, string nativeCatalog, int expectedFields) + { + using var http = CreateHttpClient(overlappingNativeColumns: true, nativeCatalog: nativeCatalog); + using var connection = CreateConnection(http); + + var schema = connection.GetTableSchema(requestedCatalog, "default", "t"); + + Assert.Equal(expectedFields, schema.FieldsList.Count); + Assert.All(schema.FieldsList, field => Assert.Equal("a", field.Name)); + } + [Fact] public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() { From 091c585b02e1470c7e68602f37374a014cb11dd2 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 17:18:38 +0000 Subject: [PATCH 18/22] fix(csharp): preserve existing SHOW metadata behavior Signed-off-by: Vu Anh Phung --- .../StatementExecution/MetadataRowReader.cs | 19 ++-- .../NativeMetadataResultBuilder.cs | 13 ++- .../StatementExecutionConnection.cs | 86 ++++------------ .../StatementExecutionStatement.cs | 3 +- .../MetadataRowReaderTests.cs | 27 ++++- .../NativeMetadataResultBuilderTests.cs | 22 +++-- ...tatementExecutionMetadataResponseTests.cs} | 99 ++++++++++++++++--- 7 files changed, 172 insertions(+), 97 deletions(-) rename csharp/test/Unit/StatementExecution/{StatementExecutionGetObjectsCatalogPatternTests.cs => StatementExecutionMetadataResponseTests.cs} (76%) diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs index 423b8b71..82a74c0d 100644 --- a/csharp/src/StatementExecution/MetadataRowReader.cs +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -93,7 +93,8 @@ internal TableRow(string catalog, string schema, string table, string tableType, internal readonly struct ColumnRow { internal ColumnRow(string? catalog, string? schema, string? table, string name, - string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement) + string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement, + bool isNative = false) { Catalog = catalog; Schema = schema; @@ -104,6 +105,7 @@ internal ColumnRow(string? catalog, string? schema, string? table, string name, Ordinal = ordinal; Default = columnDefault; IsAutoIncrement = isAutoIncrement; + IsNative = isNative; } internal string? Catalog { get; } @@ -115,6 +117,7 @@ internal ColumnRow(string? catalog, string? schema, string? table, string name, internal int Ordinal { get; } internal string? Default { get; } internal bool IsAutoIncrement { get; } + internal bool IsNative { get; } } internal static class MetadataRowReader @@ -131,7 +134,7 @@ internal static IEnumerable Catalogs(MetadataBatches result, string? pat for (int row = 0; row < batch.Length; row++) { string? catalog = result.IsNative ? native!.String("TABLE_CAT", row) : String(show, row); - if (catalog != null && MatchesCatalogPattern(pattern, catalog)) + if (catalog != null && (!result.IsNative || MatchesCatalogPattern(pattern, catalog))) yield return catalog; } } @@ -169,7 +172,8 @@ internal static IEnumerable Schemas(MetadataBatches result, string? c } internal static List Tables( - MetadataBatches result, string? catalog, IReadOnlyCollection? tableTypes = null) + MetadataBatches result, string? catalog, IReadOnlyCollection? tableTypes = null, + bool normalizeEmptyTableType = true) { var rows = new List(); foreach (var batch in result.Batches) @@ -191,8 +195,10 @@ internal static List Tables( string? table = result.IsNative ? native!.String("TABLE_NAME", row) : String(tables, row); if ((!result.IsNative && rowCatalog == null) || schema == null || table == null) continue; if (result.IsNative && !MatchesCatalog(catalog, rowCatalog)) continue; - string type = NativeMetadataResultBuilder.DefaultTableType( - result.IsNative ? native!.String("TABLE_TYPE", row) : String(types, row)); + string? serverType = result.IsNative ? native!.String("TABLE_TYPE", row) : String(types, row); + string type = result.IsNative || normalizeEmptyTableType + ? NativeMetadataResultBuilder.DefaultTableType(serverType) + : serverType ?? "TABLE"; if (tableTypes != null && !tableTypes.Contains(type)) continue; rows.Add(new TableRow(rowCatalog ?? "", schema, table, type, (result.IsNative ? native!.String("REMARKS", row) : String(remarks, row)) ?? "")); @@ -239,7 +245,8 @@ private static IEnumerable NativeColumns(RecordBatch batch, MetadataB columns.String("TABLE_NAME", row), name, typeName, columns.Integer("NULLABLE", row) == 1, checked((int)(columns.Integer("ORDINAL_POSITION", row) ?? 0)), columns.String("COLUMN_DEF", row), - string.Equals(columns.String("IS_AUTO_INCREMENT", row), "YES", StringComparison.OrdinalIgnoreCase)); + string.Equals(columns.String("IS_AUTO_INCREMENT", row), "YES", StringComparison.OrdinalIgnoreCase), + isNative: true); } } diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 56fd171b..43fdff5f 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -144,9 +144,16 @@ internal static QueryResult Build( { string? typeName = columns.String("TYPE_NAME", row); if (typeName != null) - value = name == "COLUMN_SIZE" - ? ColumnMetadataHelper.GetColumnSizeDefault(typeName) - : ColumnMetadataHelper.GetDecimalDigitsDefault(typeName); + { + string baseName = ColumnMetadataHelper.GetBaseTypeName(typeName); + if (baseName is "DECIMAL" or "NUMERIC") + value = name == "COLUMN_SIZE" + ? ColumnMetadataHelper.GetColumnSizeDefault(typeName) + : ColumnMetadataHelper.GetDecimalDigitsDefault(typeName); + else if (name == "COLUMN_SIZE" && baseName is + "CHAR" or "NCHAR" or "VARCHAR" or "NVARCHAR" or "LONGVARCHAR" or "LONGNVARCHAR" or "STRING") + value = ColumnMetadataHelper.GetColumnSizeDefault(typeName); + } } if (value.HasValue) int32.Append(checked((int)value.Value)); else int32.AppendNull(); } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index ac84a3f5..4eae36a2 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -745,9 +745,6 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string async Task> IGetObjectsDataProvider.GetCatalogsAsync(string? catalogPattern, CancellationToken cancellationToken) { - if (catalogPattern == "") - return System.Array.Empty(); - string sql = new ShowCatalogsCommand(catalogPattern).Build(); MetadataBatches result; try @@ -766,18 +763,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) { - var result = new List<(string catalog, string schema)>(); - foreach (string? catalog in await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false)) - { - result.AddRange(await GetSchemasInCatalogAsync(catalog, schemaPattern, cancellationToken).ConfigureAwait(false)); - } - return result; - } - - private async Task> GetSchemasInCatalogAsync( - string? catalog, string? schemaPattern, CancellationToken cancellationToken) - { - string sql = new ShowSchemasCommand(catalog, schemaPattern).Build(); + string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); MetadataBatches result; try @@ -790,7 +776,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin return System.Array.Empty<(string, string)>(); } - return MetadataRowReader.Schemas(result, catalog) + return MetadataRowReader.Schemas(result, catalogPattern) .Select(row => (row.Catalog, row.Schema)).ToList(); } @@ -807,23 +793,7 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin return System.Array.Empty<(string, string, string, string)>(); } - IReadOnlyList catalogs = await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false); - var result = new List(); - foreach (string? catalog in catalogs) - { - result.AddRange(await GetTablesInCatalogAsync( - catalog, schemaPattern, tableNamePattern, tableTypes, cancellationToken).ConfigureAwait(false)); - } - if (catalogs.Count > 1) - result.Sort(MetadataRowReader.CompareTables); - return result.Select(row => (row.Catalog, row.Schema, row.Table, row.TableType)).ToList(); - } - - private async Task> GetTablesInCatalogAsync( - string? catalog, string? schemaPattern, string? tableNamePattern, IReadOnlyList? tableTypes, - CancellationToken cancellationToken) - { - string sql = new ShowTablesCommand(catalog, schemaPattern, tableNamePattern).Build(); + string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); MetadataBatches result; try @@ -833,9 +803,10 @@ private async Task> GetTablesInCatalogAsync( } catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) { - return System.Array.Empty(); + return System.Array.Empty<(string, string, string, string)>(); } - return MetadataRowReader.Tables(result, catalog, tableTypes); + return MetadataRowReader.Tables(result, catalogPattern, tableTypes) + .Select(row => (row.Catalog, row.Schema, row.Table, row.TableType)).ToList(); } async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPattern, string? schemaPattern, @@ -843,30 +814,31 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - var columns = new List(); - foreach (string? catalog in await ResolveGetObjectsCatalogsAsync(catalogPattern, cancellationToken).ConfigureAwait(false)) + ColumnMetadataResult columns; + try { - try - { - var result = await ReadColumnsAsync(catalog, schemaPattern, tablePattern, - columnPattern, cancellationToken, requireExactCatalog: true).ConfigureAwait(false); - columns.AddRange(result.Rows); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - // A catalog may disappear after enumeration. - } + columns = await ReadColumnsAsync(catalogPattern, schemaPattern, tablePattern, + columnPattern, cancellationToken, requireExactCatalog: true).ConfigureAwait(false); + } + catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) + { + return; } - foreach (var column in columns) + var tablePositions = new Dictionary(); + foreach (var column in columns.Rows) { if (column.Catalog == null || column.Schema == null || column.Table == null || string.IsNullOrEmpty(column.Name)) continue; + string tableKey = $"{column.Catalog}.{column.Schema}.{column.Table}"; + tablePositions.TryGetValue(tableKey, out int position); + tablePositions[tableKey] = ++position; + int ordinal = column.IsNative ? checked(column.Ordinal + 1) : position; if (catalogMap.TryGetValue(column.Catalog, out var schemaMap) && schemaMap.TryGetValue(column.Schema, out var tableMap) && tableMap.TryGetValue(column.Table, out var tableInfo)) ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfo, column.Name, column.TypeName, checked(column.Ordinal + 1), column.Nullable, + tableInfo, column.Name, column.TypeName, ordinal, column.Nullable, columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); } } @@ -941,22 +913,6 @@ internal async Task ReadColumnsAsync( return new ColumnMetadataResult(results); } - private async Task> ResolveGetObjectsCatalogsAsync( - string? pattern, CancellationToken cancellationToken) - { - if (pattern == null || (pattern.Length > 0 && pattern.All(character => character == '%'))) - return new string?[] { null }; - if (pattern == "") - return System.Array.Empty(); - if (pattern.IndexOfAny(new[] { '%', '_', '\\' }) < 0) - return new string?[] { pattern }; - - // SHOW scopes are identifiers; resolve LIKE patterns to concrete catalog names. - IReadOnlyList catalogs = await ((IGetObjectsDataProvider)this) - .GetCatalogsAsync(pattern, cancellationToken).ConfigureAwait(false); - return catalogs.Select(catalog => (string?)catalog).ToList(); - } - internal bool EnablePKFK => _enablePKFK; internal bool EnableMultipleCatalogSupport => _enableMultipleCatalogSupport; diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 4b0fd5ea..c688a767 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1447,7 +1447,8 @@ private async Task GetTablesAsync(CancellationToken cancellationTok : null; int count = 0; - foreach (var row in MetadataRowReader.Tables(result, catalog, tableTypeFilter)) + foreach (var row in MetadataRowReader.Tables( + result, catalog, tableTypeFilter, normalizeEmptyTableType: false)) { tableCatBuilder.Append(row.Catalog); tableSchemaBuilder.Append(row.Schema); diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs index 0eaf7a09..9b500098 100644 --- a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -30,13 +30,36 @@ public class MetadataRowReaderTests [Theory] [InlineData(false)] [InlineData(true)] - public void Catalogs_ApplyTheSamePatternForBothFormats(bool native) + public void Catalogs_OnlyFilterNativeResponses(bool native) { using var batch = Strings((native ? "TABLE_CAT" : "catalog", new string?[] { "ma_catalog", "maxcatalog", "other" })); var result = new MetadataBatches(new List { batch }, native); - Assert.Equal(new[] { "ma_catalog" }, MetadataRowReader.Catalogs(result, @"ma\_%")); + string[] expected = native + ? new[] { "ma_catalog" } + : new[] { "ma_catalog", "maxcatalog", "other" }; + Assert.Equal(expected, MetadataRowReader.Catalogs(result, @"ma\_%")); + } + + [Theory] + [InlineData(false, "")] + [InlineData(true, "TABLE")] + public void ShowTables_EmptyTypeKeepsExistingCallSiteBehavior(bool normalizeEmptyTableType, string expectedType) + { + using var batch = Strings( + ("catalogName", new string?[] { "main" }), + ("namespace", new string?[] { "default" }), + ("tableName", new string?[] { "t" }), + ("tableType", new string?[] { "" })); + var result = new MetadataBatches(new List { batch }, false); + + var row = Assert.Single(MetadataRowReader.Tables( + result, "main", normalizeEmptyTableType: normalizeEmptyTableType)); + Assert.Equal(expectedType, row.TableType); + var filtered = MetadataRowReader.Tables( + result, "main", new[] { "TABLE" }, normalizeEmptyTableType: normalizeEmptyTableType); + Assert.Equal(normalizeEmptyTableType ? 1 : 0, filtered.Count); } [Fact] diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index 8ee73517..be3c5eb5 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -28,8 +28,15 @@ namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution { public class NativeMetadataResultBuilderTests { - [Fact] - public async Task GetColumns_NormalizesSizeAndScaleAndKeepsFlatOrdinalZeroBased() + [Theory] + [InlineData("DECIMAL(10,2)", 0, 0, 10, 2)] + [InlineData("INT", 10, 0, 10, 0)] + [InlineData("FLOAT", 7, 7, 7, 7)] + [InlineData("BINARY", 99, 3, 99, 3)] + [InlineData("VARCHAR(42)", 99, 3, 42, 3)] + [InlineData("CHAR(4)", 99, 3, 4, 3)] + public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( + string typeName, int columnSize, int decimalDigits, int expectedSize, int expectedScale) { var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); Assert.Equal(24, target.FieldsList.Count); @@ -46,13 +53,14 @@ public async Task GetColumns_NormalizesSizeAndScaleAndKeepsFlatOrdinalZeroBased( else if (field.Name == "TABLE_CAT") arrays.Add(new StringArray.Builder().AppendNull().Build()); else if (field.DataType.TypeId == ArrowTypeId.String) - arrays.Add(new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? "DECIMAL(10,2)" : "value").Build()); + arrays.Add(new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? typeName : "value").Build()); else if (field.DataType.TypeId == ArrowTypeId.Int8) arrays.Add(new Int8Array.Builder().Append(0).Build()); else if (field.DataType.TypeId == ArrowTypeId.Int16) arrays.Add(new Int16Array.Builder().Append(0).Build()); else - arrays.Add(new Int32Array.Builder().Append(0).Build()); + arrays.Add(new Int32Array.Builder().Append(field.Name == "COLUMN_SIZE" ? columnSize + : field.Name == "DECIMAL_DIGITS" ? decimalDigits : 0).Build()); } using var nativeBatch = new RecordBatch(source, arrays.ToArray(), 1); @@ -63,11 +71,11 @@ public async Task GetColumns_NormalizesSizeAndScaleAndKeepsFlatOrdinalZeroBased( using var batch = await reader.ReadNextRecordBatchAsync(); Assert.NotNull(batch); - Assert.Equal(10, ((Int32Array)batch.Column(6)).GetValue(0)); - Assert.Equal(2, ((Int32Array)batch.Column(8)).GetValue(0)); + Assert.Equal(expectedSize, ((Int32Array)batch.Column(6)).GetValue(0)); + Assert.Equal(expectedScale, ((Int32Array)batch.Column(8)).GetValue(0)); Assert.Equal(0, ((Int32Array)batch.Column(16)).GetValue(0)); Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); - Assert.Equal("DECIMAL", ((StringArray)batch.Column(23)).GetString(0)); + Assert.Equal(ColumnMetadataHelper.GetBaseTypeName(typeName), ((StringArray)batch.Column(23)).GetString(0)); } [Fact] diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs similarity index 76% rename from csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs rename to csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs index ab91d4db..bd58e4b8 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionGetObjectsCatalogPatternTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs @@ -38,7 +38,7 @@ namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution { - public class StatementExecutionGetObjectsCatalogPatternTests + public class StatementExecutionMetadataResponseTests { private static readonly string[] s_catalogs = { "main", "marketing", "other", "foo_bar", "fooxbar" }; @@ -71,11 +71,11 @@ public class StatementExecutionGetObjectsCatalogPatternTests [Theory] [MemberData(nameof(CatalogPatterns))] - public async Task GetObjects_ResolvesCatalogPatterns( + public async Task GetObjects_PreservesExistingCatalogScope( int mode, AdbcConnection.GetObjectsDepth depth, string? pattern, string[] expectedCatalogs) { List statements = new List(); - using HttpClient http = CreateHttpClient(mode, statements); + using HttpClient http = CreateHttpClient(mode, statements, expectedCatalogs); using StatementExecutionConnection connection = CreateConnection(http); using IArrowArrayStream stream = connection.GetObjects(depth, pattern, "default", "t1", null, null); using RecordBatch? batch = await stream.ReadNextRecordBatchAsync(); @@ -86,6 +86,12 @@ public async Task GetObjects_ResolvesCatalogPatterns( catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); foreach (AdbcCatalog catalog in catalogs) { + if (pattern != null && !string.Equals(pattern, catalog.Name, StringComparison.OrdinalIgnoreCase)) + { + Assert.Empty(catalog.DbSchemas!); + continue; + } + AdbcDbSchema schema = Assert.Single(catalog.DbSchemas!); Assert.Equal("default", schema.Name); AdbcTable table = Assert.Single(schema.Tables!); @@ -98,10 +104,20 @@ public async Task GetObjects_ResolvesCatalogPatterns( } } - if (pattern == "") - Assert.Empty(statements); - if (pattern == "main") - Assert.Single(statements, sql => sql.StartsWith("SHOW CATALOGS", StringComparison.Ordinal)); + int catalogQueries = pattern == null && depth == AdbcConnection.GetObjectsDepth.All ? 2 : 1; + Assert.Equal(catalogQueries, + statements.Count(sql => sql.StartsWith("SHOW CATALOGS", StringComparison.Ordinal))); + if (pattern != null) + { + string scope = $"`{pattern.ToLowerInvariant()}`"; + Assert.Contains(statements, sql => sql.StartsWith("SHOW SCHEMAS", StringComparison.Ordinal) + && sql.Contains(scope)); + Assert.Contains(statements, sql => sql.StartsWith("SHOW TABLES", StringComparison.Ordinal) + && sql.Contains(scope)); + if (depth == AdbcConnection.GetObjectsDepth.All) + Assert.Contains(statements, sql => sql.StartsWith("SHOW COLUMNS", StringComparison.Ordinal) + && sql.Contains(scope)); + } } [Theory] @@ -172,6 +188,40 @@ public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdin } } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetObjects_SkippedShowColumnsDoNotChangeOrdinals(bool native) + { + using HttpClient http = CreateHttpClient( + native ? 1 : 0, new List(), new[] { "main" }, prependEmptyShowColumn: true); + using StatementExecutionConnection connection = CreateConnection(http); + using IArrowArrayStream stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, "main", "default", "t1", null, null); + using RecordBatch batch = (await stream.ReadNextRecordBatchAsync())!; + var catalog = Assert.Single(GetObjectsParser.ParseCatalog(batch, null)); + var schema = Assert.Single(catalog.DbSchemas!); + var table = Assert.Single(schema.Tables!); + var column = Assert.Single(table.Columns!); + Assert.Equal("a", column.Name); + Assert.Equal(1, column.OrdinalPosition); + + using var statement = connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.EscapePatternWildcards, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SetOption(ApacheParameters.SchemaName, "default"); + statement.SetOption(ApacheParameters.TableName, "t1"); + statement.SqlQuery = "GetColumns"; + using var flatStream = statement.ExecuteQuery().Stream!; + using var flatBatch = (await flatStream.ReadNextRecordBatchAsync())!; + Assert.Equal(native ? 1 : 2, flatBatch.Length); + Assert.Equal(native ? "a" : "", ((StringArray)flatBatch.Column("COLUMN_NAME")).GetString(0)); + Assert.Equal(0, ((Int32Array)flatBatch.Column("ORDINAL_POSITION")).GetValue(0)); + if (!native) + Assert.Equal(1, ((Int32Array)flatBatch.Column("ORDINAL_POSITION")).GetValue(1)); + } + [Theory] [InlineData(0, false, false)] [InlineData(1, false, true)] @@ -190,7 +240,7 @@ public void NativeResponseAssertions_CheckInternalMetadataStatements( if (getObjects) { using var stream = connection.GetObjects( - AdbcConnection.GetObjectsDepth.All, "ma%", "default", "t1", null, null); + AdbcConnection.GetObjectsDepth.All, null, "default", "t1", null, null); commands = new[] { "SHOW CATALOGS", "SHOW SCHEMAS", "SHOW TABLES", "SHOW COLUMNS" }; } else @@ -224,7 +274,8 @@ private static StatementExecutionConnection CreateConnection(HttpClient http, bo : new StatementExecutionConnection(properties, http); } - private static HttpClient CreateHttpClient(int mode, List statements) + private static HttpClient CreateHttpClient( + int mode, List statements, string[]? matchingCatalogs = null, bool prependEmptyShowColumn = false) { Mock handler = new Mock(); handler.Protected() @@ -242,11 +293,27 @@ private static HttpClient CreateHttpClient(int mode, List statements) string sql = json.RootElement.GetProperty("statement").GetString()!; statements.Add(sql); string operation = request.Headers.GetValues("x-databricks-metadata-operation-type").Single(); - string? requestedCatalog = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs" - ? null : s_catalogs.Single(catalog => sql.Contains($"`{catalog}`")); + bool allCatalogs = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs"; + string? requestedCatalog = allCatalogs + ? null : s_catalogs.SingleOrDefault(catalog => sql.Contains($"`{catalog}`")); + if (!allCatalogs && requestedCatalog == null) + { + return Response(JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new + { + state = "FAILED", + sql_state = "42704", + error = new { error_code = "CATALOG_NOT_FOUND", message = "Catalog not found" }, + }, + })); + } bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || (mode == 3 && (operation != "GetColumns" || requestedCatalog == "main")); string[] catalogs = requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; + if (!native && operation == "GetCatalogs") + catalogs = matchingCatalogs ?? s_catalogs; if (native && requestedCatalog != null && operation is "GetTables" or "GetColumns") catalogs = catalogs.Concat(new[] { requestedCatalog == "other" ? "main" : "other" }).ToArray(); @@ -254,6 +321,11 @@ private static HttpClient CreateHttpClient(int mode, List statements) using MemoryStream raw = new MemoryStream(); using (ArrowStreamWriter writer = new ArrowStreamWriter(raw, batch.Schema)) { + if (!native && operation == "GetColumns" && prependEmptyShowColumn) + { + using var empty = CreateBatch(operation, false, catalogs, requestedCatalog, columnName: ""); + writer.WriteRecordBatch(empty); + } writer.WriteRecordBatch(batch); writer.WriteEnd(); } @@ -286,7 +358,8 @@ private static HttpClient CreateHttpClient(int mode, List statements) private static HttpResponseMessage Response(string body) => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; - private static RecordBatch CreateBatch(string operation, bool native, string[] catalogs, string? requestedCatalog) + private static RecordBatch CreateBatch( + string operation, bool native, string[] catalogs, string? requestedCatalog, string columnName = "a") { Schema schema; if (native) @@ -325,7 +398,7 @@ private static RecordBatch CreateBatch(string operation, bool native, string[] c "TABLE_SCHEM" or "databaseName" or "namespace" => "default", "TABLE_NAME" or "tableName" => "t1", "TABLE_TYPE" or "tableType" => "TABLE", - "COLUMN_NAME" or "col_name" => "a", + "COLUMN_NAME" or "col_name" => columnName, "TYPE_NAME" or "columnType" => "INT", "COLUMN_DEF" => "7", "IS_AUTO_INCREMENT" => "YES", From 1d73109faa16ab67061b01dfc7e9643d0587df73 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 17:32:19 +0000 Subject: [PATCH 19/22] test(csharp): simplify native metadata coverage Signed-off-by: Vu Anh Phung --- ...entExecutionMetadataObjectNotFoundTests.cs | 39 ------------------- ...StatementExecutionMetadataResponseTests.cs | 18 ++++----- .../StatementExecutionNativeCatalogTests.cs | 11 ------ 3 files changed, 9 insertions(+), 59 deletions(-) diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 33608393..6d15e4ce 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -169,45 +169,6 @@ private static StatementExecutionStatement CreateMetadataStatement( connection: connection); } - [Fact] - public async Task MetadataConnection_AlwaysRequestsNativeMetadata() - { - bool hasSyncHeader = false; - bool hasOperationHeader = false; - bool hasRequireHeader = false; - var handler = new Mock(); - handler.Protected() - .Setup>("SendAsync", - ItExpr.IsAny(), ItExpr.IsAny()) - .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => - { - if (request.Method == HttpMethod.Post && request.RequestUri?.AbsolutePath.EndsWith("/sql/statements") == true) - { - hasSyncHeader = request.Headers.Contains("x-databricks-sea-can-run-fully-sync"); - hasOperationHeader = request.Headers.Contains("x-databricks-metadata-operation-type"); - hasRequireHeader = request.Headers.Contains("x-databricks-require-thrift-native-metadata"); - } - string body = request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true - ? "{\"session_id\":\"session-1\"}" - : "{\"statement_id\":\"stmt-1\",\"status\":{\"state\":\"SUCCEEDED\"}}"; - return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; - }); - using var http = new HttpClient(handler.Object); - var properties = new Dictionary - { - [SparkParameters.HostName] = "test.databricks.com", - [DatabricksParameters.WarehouseId] = "wh-1", - [SparkParameters.AccessToken] = "token", - }; - using var connection = new StatementExecutionConnection(properties, http); - - await connection.ExecuteMetadataCommandAsync("SHOW CATALOGS", MetadataOperation.GetCatalogs, CancellationToken.None); - - Assert.True(hasSyncHeader); - Assert.True(hasOperationHeader); - Assert.True(hasRequireHeader); - } - // ─── Issue #525: `%` match-all catalog wildcard ────────────────────────────── // The `%` SQL-LIKE wildcard must mean "all catalogs" on the SEA path, exactly as // Thrift treats it, rather than being passed through as a literal backtick-quoted diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs index bd58e4b8..3754679a 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs @@ -58,15 +58,15 @@ public class StatementExecutionMetadataResponseTests (@"foo\_bar", new[] { "foo_bar" }), ("foo_bar", new[] { "foo_bar", "fooxbar" }), }; - foreach (int mode in new[] { 0, 1, 2, 3 }) + foreach (int mode in new[] { 0, 1 }) { - foreach (AdbcConnection.GetObjectsDepth depth in new[] - { AdbcConnection.GetObjectsDepth.Tables, AdbcConnection.GetObjectsDepth.All }) - { - foreach (var (pattern, matches) in patterns) - yield return new object?[] { mode, depth, pattern, matches }; - } + foreach (var (pattern, matches) in patterns) + yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.All, pattern, matches }; + foreach (var (pattern, matches) in patterns.Where(entry => entry.Pattern == null || entry.Pattern == "main")) + yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.Tables, pattern, matches }; } + foreach (int mode in new[] { 2, 3 }) + yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.All, null, s_catalogs }; } [Theory] @@ -225,8 +225,6 @@ public async Task GetObjects_SkippedShowColumnsDoNotChangeOrdinals(bool native) [Theory] [InlineData(0, false, false)] [InlineData(1, false, true)] - [InlineData(2, false, false)] - [InlineData(3, false, true)] [InlineData(0, true, false)] [InlineData(1, true, true)] [InlineData(2, true, false)] @@ -288,6 +286,8 @@ private static HttpClient CreateHttpClient( if (request.Method != HttpMethod.Post) return new HttpResponseMessage(HttpStatusCode.OK); + Assert.Equal("true", Assert.Single(request.Headers.GetValues("x-databricks-sea-can-run-fully-sync"))); + Assert.Equal("true", Assert.Single(request.Headers.GetValues("x-databricks-require-thrift-native-metadata"))); using JsonDocument json = JsonDocument.Parse( request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); string sql = json.RootElement.GetProperty("statement").GetString()!; diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 4b1be8f2..2297c8eb 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -163,17 +163,6 @@ private static StatementExecutionConnection CreateConnection(HttpClient http) return new StatementExecutionConnection(properties, http); } - [Fact] - public async Task NativeCatalogs_AreFilteredForGetObjects() - { - using var http = CreateHttpClient(); - using var connection = CreateConnection(http); - - var catalogs = await ((IGetObjectsDataProvider)connection).GetCatalogsAsync("main", CancellationToken.None); - - Assert.Equal(new[] { "main" }, catalogs); - } - [Fact] public async Task GetObjects_CatalogDepth_FiltersNativeCatalogs() { From ea23639887891591fd8878aa0eb66d10ed00ea1b Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Wed, 7 Oct 2026 19:48:42 +0000 Subject: [PATCH 20/22] fix(csharp): match Thrift native metadata null defaults Signed-off-by: Vu Anh Phung --- .../StatementExecution/NativeMetadataResultBuilder.cs | 2 ++ .../NativeMetadataResultBuilderTests.cs | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 43fdff5f..19d337b6 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -142,6 +142,8 @@ internal static QueryResult Build( long? value = columns.Integer(name, row); if (operation == MetadataOperation.GetColumns && (name is "COLUMN_SIZE" or "DECIMAL_DIGITS")) { + // Thrift defaults null sizes and digits to zero before type-specific overrides. + value ??= 0; string? typeName = columns.String("TYPE_NAME", row); if (typeName != null) { diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index be3c5eb5..cf8bb821 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -35,8 +35,13 @@ public class NativeMetadataResultBuilderTests [InlineData("BINARY", 99, 3, 99, 3)] [InlineData("VARCHAR(42)", 99, 3, 42, 3)] [InlineData("CHAR(4)", 99, 3, 4, 3)] + [InlineData("DOUBLE", null, null, 0, 0)] + [InlineData("DOUBLE", 15, null, 15, 0)] + [InlineData("DOUBLE", null, 15, 0, 15)] + [InlineData("DECIMAL(10,2)", null, null, 10, 2)] + [InlineData("VARCHAR(42)", null, null, 42, 0)] public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( - string typeName, int columnSize, int decimalDigits, int expectedSize, int expectedScale) + string typeName, int? columnSize, int? decimalDigits, int expectedSize, int expectedScale) { var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); Assert.Equal(24, target.FieldsList.Count); @@ -58,6 +63,8 @@ public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( arrays.Add(new Int8Array.Builder().Append(0).Build()); else if (field.DataType.TypeId == ArrowTypeId.Int16) arrays.Add(new Int16Array.Builder().Append(0).Build()); + else if (field.Name == "BUFFER_LENGTH") + arrays.Add(new Int32Array.Builder().AppendNull().Build()); else arrays.Add(new Int32Array.Builder().Append(field.Name == "COLUMN_SIZE" ? columnSize : field.Name == "DECIMAL_DIGITS" ? decimalDigits : 0).Build()); @@ -72,6 +79,7 @@ public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( Assert.NotNull(batch); Assert.Equal(expectedSize, ((Int32Array)batch.Column(6)).GetValue(0)); + Assert.True(batch.Column(7).IsNull(0)); Assert.Equal(expectedScale, ((Int32Array)batch.Column(8)).GetValue(0)); Assert.Equal(0, ((Int32Array)batch.Column(16)).GetValue(0)); Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); From 92db6139a06a85c594a6e6316b932574c39b25a1 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 9 Oct 2026 07:12:26 +0000 Subject: [PATCH 21/22] fix(csharp): unify native metadata decoding and catalog filters Signed-off-by: Vu Anh Phung --- .../ColumnMetadataHelper.cs | 25 +- .../StatementExecution/MetadataRowReader.cs | 446 ++++++++++++++---- .../NativeMetadataColumns.cs | 28 +- .../NativeMetadataResultBuilder.cs | 193 +++----- .../StatementExecutionConnection.cs | 141 +++--- .../StatementExecutionStatement.cs | 292 +++--------- csharp/test/ColumnMetadataHelperTests.cs | 16 + .../MetadataRowReaderTests.cs | 197 +++++++- .../NativeMetadataResultBuilderTests.cs | 71 ++- ...entExecutionMetadataObjectNotFoundTests.cs | 52 +- ...StatementExecutionMetadataResponseTests.cs | 34 +- .../StatementExecutionNativeCatalogTests.cs | 13 +- 12 files changed, 871 insertions(+), 637 deletions(-) diff --git a/csharp/src/StatementExecution/ColumnMetadataHelper.cs b/csharp/src/StatementExecution/ColumnMetadataHelper.cs index 93927a88..cfe103f9 100644 --- a/csharp/src/StatementExecution/ColumnMetadataHelper.cs +++ b/csharp/src/StatementExecution/ColumnMetadataHelper.cs @@ -184,6 +184,24 @@ internal static string GetBaseTypeName(string typeName) }; } + internal static (long ColumnSize, long DecimalDigits) NormalizePrecisionAndScale( + string? typeName, long? columnSize, long? decimalDigits) + { + if (typeName != null) + { + string baseName = GetBaseTypeName(typeName); + if (baseName is "DECIMAL" or "NUMERIC") + { + columnSize = GetColumnSizeDefault(typeName); + decimalDigits = GetDecimalDigitsDefault(typeName); + } + else if (s_charBaseTypes.Contains(baseName)) + columnSize = GetColumnSizeDefault(typeName); + } + // Thrift defaults null sizes and digits to zero before type-specific overrides. + return (columnSize ?? 0, decimalDigits ?? 0); + } + internal static int? GetBufferLength(string typeName) { string baseName = GetBaseTypeName(typeName); @@ -249,9 +267,10 @@ internal static void PopulateTableInfoFromTypeName( tableInfo.TypeName.Add(typeName); tableInfo.ColType.Add(GetDataTypeCode(typeName)); tableInfo.BaseTypeName.Add(GetBaseTypeName(typeName)); - tableInfo.Precision.Add(GetColumnSizeDefault(typeName)); - int? scale = GetDecimalDigitsDefault(typeName); - tableInfo.Scale.Add(scale.HasValue ? (short)scale.Value : null); + var precisionAndScale = NormalizePrecisionAndScale( + typeName, GetColumnSizeDefault(typeName), GetDecimalDigitsDefault(typeName)); + tableInfo.Precision.Add(checked((int)precisionAndScale.ColumnSize)); + tableInfo.Scale.Add((short)precisionAndScale.DecimalDigits); tableInfo.OrdinalPosition.Add(ordinalPosition); tableInfo.Nullable.Add(isNullable ? (short)1 : (short)0); tableInfo.IsNullable.Add(isNullable ? "YES" : "NO"); diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs index 82a74c0d..bd8c2438 100644 --- a/csharp/src/StatementExecution/MetadataRowReader.cs +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -25,37 +25,49 @@ namespace AdbcDrivers.Databricks.StatementExecution { + internal readonly struct CatalogFilter + { + private CatalogFilter(string? value, bool isPattern) + { + Value = value; + IsPattern = isPattern; + } + + internal string? Value { get; } + internal bool IsPattern { get; } + + internal static CatalogFilter Exact(string? catalog) => new(catalog, false); + internal static CatalogFilter Pattern(string? pattern) => new(pattern, true); + } + internal sealed class MetadataBatches { internal MetadataBatches(List batches, bool isNative, - string? sourceCatalog = null, bool requireExactCatalog = false) + string? sourceCatalog = null, CatalogFilter catalogFilter = default) { Batches = batches; IsNative = isNative; SourceCatalog = sourceCatalog; - RequireExactCatalog = requireExactCatalog; + CatalogFilter = catalogFilter; } internal List Batches { get; } internal bool IsNative { get; } internal string? SourceCatalog { get; } - internal bool RequireExactCatalog { get; } + internal CatalogFilter CatalogFilter { get; } } internal sealed class ColumnMetadataResult { - internal ColumnMetadataResult(IReadOnlyList results) + internal ColumnMetadataResult(IReadOnlyList results, bool requireTableIdentifiers = false) { - Batches = results.SelectMany(result => result.Batches).ToList(); - SourceCatalogs = results.SelectMany(result => - Enumerable.Repeat(result.SourceCatalog, result.Batches.Count)).ToList(); - IsNative = Batches.Count > 0 && + Results = results; + IsNative = results.Any(result => result.Batches.Count > 0) && results.Where(result => result.Batches.Count > 0).All(result => result.IsNative); - Rows = MetadataRowReader.Columns(results).ToList(); + Rows = MetadataRowReader.Columns(results, requireTableIdentifiers).ToList(); } - internal IReadOnlyList Batches { get; } - internal IReadOnlyList SourceCatalogs { get; } + internal IReadOnlyList Results { get; } internal bool IsNative { get; } internal IReadOnlyList Rows { get; } } @@ -93,8 +105,7 @@ internal TableRow(string catalog, string schema, string table, string tableType, internal readonly struct ColumnRow { internal ColumnRow(string? catalog, string? schema, string? table, string name, - string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement, - bool isNative = false) + string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement) { Catalog = catalog; Schema = schema; @@ -105,7 +116,6 @@ internal ColumnRow(string? catalog, string? schema, string? table, string name, Ordinal = ordinal; Default = columnDefault; IsAutoIncrement = isAutoIncrement; - IsNative = isNative; } internal string? Catalog { get; } @@ -117,136 +127,258 @@ internal ColumnRow(string? catalog, string? schema, string? table, string name, internal int Ordinal { get; } internal string? Default { get; } internal bool IsAutoIncrement { get; } - internal bool IsNative { get; } + } + + internal readonly struct PrimaryKeyRow + { + internal PrimaryKeyRow(string catalog, string schema, string table, string column, int sequence, string name) + { + Catalog = catalog; + Schema = schema; + Table = table; + Column = column; + Sequence = sequence; + Name = name; + } + + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string Column { get; } + internal int Sequence { get; } + internal string Name { get; } + } + + internal readonly struct ForeignKeyRow + { + internal ForeignKeyRow(string parentCatalog, string parentSchema, string parentTable, string parentColumn, + string catalog, string schema, string table, string column, int sequence, + int updateRule, int deleteRule, string name, string? parentName, int deferrability) + { + ParentCatalog = parentCatalog; + ParentSchema = parentSchema; + ParentTable = parentTable; + ParentColumn = parentColumn; + Catalog = catalog; + Schema = schema; + Table = table; + Column = column; + Sequence = sequence; + UpdateRule = updateRule; + DeleteRule = deleteRule; + Name = name; + ParentName = parentName; + Deferrability = deferrability; + } + + internal string ParentCatalog { get; } + internal string ParentSchema { get; } + internal string ParentTable { get; } + internal string ParentColumn { get; } + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string Column { get; } + internal int Sequence { get; } + internal int UpdateRule { get; } + internal int DeleteRule { get; } + internal string Name { get; } + internal string? ParentName { get; } + internal int Deferrability { get; } + } + + internal readonly struct NativeMetadataRow + { + internal NativeMetadataRow(NativeMetadataColumns columns, int index, string? sourceCatalog) + { + Columns = columns; + Index = index; + SourceCatalog = sourceCatalog; + } + + private NativeMetadataColumns Columns { get; } + private int Index { get; } + internal string? SourceCatalog { get; } + internal string? String(string name) => Columns.String(name, Index); + internal long? Integer(string name) => Columns.Integer(name, Index); } internal static class MetadataRowReader { - internal static IEnumerable Catalogs(MetadataBatches result, string? pattern = null) + internal static List NativeRows( + IEnumerable results, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) { - foreach (var batch in result.Batches) + string? catalogField = operation switch { - var native = result.IsNative - ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs) - : null; - var show = result.IsNative ? null : TryGetColumn(batch, "catalog"); - if (!result.IsNative && show == null) continue; - for (int row = 0; row < batch.Length; row++) + MetadataOperation.GetCatalogs => "TABLE_CAT", + MetadataOperation.GetSchemas => "TABLE_CATALOG", + MetadataOperation.GetTables or MetadataOperation.GetColumns => "TABLE_CAT", + _ => null, + }; + var rows = new List(); + foreach (var result in results) + { + // A wildcard scope cannot supply a missing native catalog identifier. + string? sourceCatalog = result.CatalogFilter.IsPattern + ? LiteralCatalog(result.SourceCatalog) : result.SourceCatalog; + foreach (var batch in result.Batches) { - string? catalog = result.IsNative ? native!.String("TABLE_CAT", row) : String(show, row); - if (catalog != null && (!result.IsNative || MatchesCatalogPattern(pattern, catalog))) - yield return catalog; + var columns = new NativeMetadataColumns(batch, schema, operation); + for (int index = 0; index < batch.Length; index++) + { + var row = new NativeMetadataRow(columns, index, sourceCatalog); + if (catalogField != null && + !MatchesCatalog(result.CatalogFilter, row.String(catalogField) ?? row.SourceCatalog)) + continue; + if (operation == MetadataOperation.GetTables && tableTypes != null && + !tableTypes.Contains(DefaultTableType(row.String("TABLE_TYPE")))) + continue; + if (operation == MetadataOperation.GetCrossReference && + !MatchesParent(parentCatalog, parentSchema, parentTable, + row.String("PKTABLE_CAT"), row.String("PKTABLE_SCHEM"), row.String("PKTABLE_NAME"))) + continue; + rows.Add(row); + } } } + + // Thrift orders tables by type, catalog, schema, then name. + if (operation == MetadataOperation.GetTables) + rows.Sort((left, right) => CompareTables( + (left.String("TABLE_CAT") ?? left.SourceCatalog, left.String("TABLE_SCHEM"), + left.String("TABLE_NAME"), left.String("TABLE_TYPE")), + (right.String("TABLE_CAT") ?? right.SourceCatalog, right.String("TABLE_SCHEM"), + right.String("TABLE_NAME"), right.String("TABLE_TYPE")))); + return rows; } - internal static IEnumerable Schemas(MetadataBatches result, string? catalog) + internal static IEnumerable Catalogs(MetadataBatches result) { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs)) + if (row.String("TABLE_CAT") is string catalog) yield return catalog; + yield break; + } + foreach (var batch in result.Batches) { - if (result.IsNative) - { - var native = new NativeMetadataColumns( - batch, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); - for (int row = 0; row < batch.Length; row++) - { - string? schema = native.String("TABLE_SCHEM", row); - string? rowCatalog = native.String("TABLE_CATALOG", row) ?? catalog; - if (schema != null && MatchesCatalog(catalog, rowCatalog)) - yield return new SchemaRow(rowCatalog ?? "", schema); - } - continue; - } + var catalogs = TryGetColumn(batch, "catalog"); + if (catalogs == null) continue; + for (int row = 0; row < batch.Length; row++) + if (String(catalogs, row) is string catalog) yield return catalog; + } + } + internal static IEnumerable Schemas(MetadataBatches result) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas)) + if (row.String("TABLE_SCHEM") is string schema) + yield return new SchemaRow(row.String("TABLE_CATALOG") ?? row.SourceCatalog ?? "", schema); + yield break; + } + + foreach (var batch in result.Batches) + { // Scoped SHOW SCHEMAS omits the catalog column. var schemas = batch.Column(0) as StringArray; - var catalogs = catalog == null ? batch.Column(1) as StringArray : null; + var catalogs = result.SourceCatalog == null ? batch.Column(1) as StringArray : null; if (schemas == null) continue; for (int row = 0; row < batch.Length; row++) { string? schema = String(schemas, row); if (schema != null) - yield return new SchemaRow(String(catalogs, row) ?? catalog ?? "", schema); + yield return new SchemaRow(String(catalogs, row) ?? result.SourceCatalog ?? "", schema); } } } internal static List Tables( - MetadataBatches result, string? catalog, IReadOnlyCollection? tableTypes = null, + MetadataBatches result, IReadOnlyCollection? tableTypes = null, bool normalizeEmptyTableType = true) { var rows = new List(); + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, tableTypes)) + { + string? schema = row.String("TABLE_SCHEM"); + string? table = row.String("TABLE_NAME"); + if (schema == null || table == null) continue; + rows.Add(new TableRow(row.String("TABLE_CAT") ?? row.SourceCatalog ?? "", schema, table, + DefaultTableType(row.String("TABLE_TYPE")), row.String("REMARKS") ?? "")); + } + return rows; + } foreach (var batch in result.Batches) { - var native = result.IsNative - ? new NativeMetadataColumns(batch, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables) - : null; - var catalogs = result.IsNative ? null : TryGetColumn(batch, "catalogName"); - var schemas = result.IsNative ? null : TryGetColumn(batch, "namespace"); - var tables = result.IsNative ? null : TryGetColumn(batch, "tableName"); - var types = result.IsNative ? null : TryGetColumn(batch, "tableType"); - var remarks = result.IsNative ? null : TryGetColumn(batch, "remarks"); - if (!result.IsNative && (catalogs == null || schemas == null || tables == null)) continue; + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var types = TryGetColumn(batch, "tableType"); + var remarks = TryGetColumn(batch, "remarks"); + if (catalogs == null || schemas == null || tables == null) continue; for (int row = 0; row < batch.Length; row++) { - string? rowCatalog = result.IsNative ? native!.String("TABLE_CAT", row) ?? catalog : String(catalogs, row); - string? schema = result.IsNative ? native!.String("TABLE_SCHEM", row) : String(schemas, row); - string? table = result.IsNative ? native!.String("TABLE_NAME", row) : String(tables, row); - if ((!result.IsNative && rowCatalog == null) || schema == null || table == null) continue; - if (result.IsNative && !MatchesCatalog(catalog, rowCatalog)) continue; - string? serverType = result.IsNative ? native!.String("TABLE_TYPE", row) : String(types, row); - string type = result.IsNative || normalizeEmptyTableType - ? NativeMetadataResultBuilder.DefaultTableType(serverType) + string? rowCatalog = String(catalogs, row); + string? schema = String(schemas, row); + string? table = String(tables, row); + if (rowCatalog == null || schema == null || table == null) continue; + string? serverType = String(types, row); + string type = normalizeEmptyTableType + ? DefaultTableType(serverType) : serverType ?? "TABLE"; if (tableTypes != null && !tableTypes.Contains(type)) continue; rows.Add(new TableRow(rowCatalog ?? "", schema, table, type, - (result.IsNative ? native!.String("REMARKS", row) : String(remarks, row)) ?? "")); + String(remarks, row) ?? "")); } } - // Thrift orders tables by type, catalog, schema, then name. - if (result.IsNative) rows.Sort(CompareTables); return rows; } - internal static IEnumerable Columns(IEnumerable results) + internal static IEnumerable Columns(IEnumerable results, + bool requireTableIdentifiers = false) { var positions = new Dictionary(); foreach (var result in results) { - foreach (var batch in result.Batches) + var rows = result.IsNative ? NativeColumns(result) + : result.Batches.SelectMany(batch => ShowColumns(batch, positions)); + foreach (var row in rows) { - var rows = result.IsNative ? NativeColumns(batch, result) : ShowColumns(batch, positions); - foreach (var row in rows) - { - string key = $"{row.Catalog}.{row.Schema}.{row.Table}"; - positions.TryGetValue(key, out int position); - positions[key] = position + 1; - yield return row; - } + // GetObjects skips incomplete SHOW rows before assigning ordinals. + if (requireTableIdentifiers && (row.Catalog == null || row.Schema == null || + row.Table == null || string.IsNullOrEmpty(row.Name))) continue; + string key = $"{row.Catalog}.{row.Schema}.{row.Table}"; + positions.TryGetValue(key, out int position); + positions[key] = position + 1; + yield return row; } } } - private static IEnumerable NativeColumns(RecordBatch batch, MetadataBatches result) + private static IEnumerable NativeColumns(MetadataBatches result) { - var columns = new NativeMetadataColumns( - batch, MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); - for (int row = 0; row < batch.Length; row++) + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns)) { - string? catalog = columns.String("TABLE_CAT", row) ?? result.SourceCatalog; - // A quoted catalog can still expand as a native LIKE pattern. - if (result.RequireExactCatalog && !MatchesCatalog(result.SourceCatalog, catalog)) continue; - string? name = columns.String("COLUMN_NAME", row); - string? typeName = columns.String("TYPE_NAME", row); + string? name = row.String("COLUMN_NAME"); + string? typeName = row.String("TYPE_NAME"); if (name == null || typeName == null) continue; // Both decoders expose zero-based ordinals; GetObjects adds its required offset. - yield return new ColumnRow(catalog, columns.String("TABLE_SCHEM", row), - columns.String("TABLE_NAME", row), name, typeName, columns.Integer("NULLABLE", row) == 1, - checked((int)(columns.Integer("ORDINAL_POSITION", row) ?? 0)), - columns.String("COLUMN_DEF", row), - string.Equals(columns.String("IS_AUTO_INCREMENT", row), "YES", StringComparison.OrdinalIgnoreCase), - isNative: true); + yield return new ColumnRow(row.String("TABLE_CAT") ?? row.SourceCatalog, row.String("TABLE_SCHEM"), + row.String("TABLE_NAME"), name, typeName, row.Integer("NULLABLE") == 1, + checked((int)(row.Integer("ORDINAL_POSITION") ?? 0)), + row.String("COLUMN_DEF"), + string.Equals(row.String("IS_AUTO_INCREMENT"), "YES", StringComparison.OrdinalIgnoreCase)); } } @@ -274,18 +406,104 @@ private static IEnumerable ShowColumns(RecordBatch batch, Dictionary< } } - internal static int CompareTables(TableRow left, TableRow right) - => CompareTables( - (left.Catalog, left.Schema, left.Table, left.TableType), - (right.Catalog, right.Schema, right.Table, right.TableType)); + internal static IEnumerable PrimaryKeys( + MetadataBatches result, string catalog, string schema, string table) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys)) + if (row.String("COLUMN_NAME") is string column) + yield return new PrimaryKeyRow(row.String("TABLE_CAT") ?? catalog, + row.String("TABLE_SCHEM") ?? schema, row.String("TABLE_NAME") ?? table, + column, checked((int)(row.Integer("KEQ_SEQ") ?? 0)), row.String("PK_NAME") ?? ""); + yield break; + } + + int sequence = 0; + foreach (var batch in result.Batches) + { + var columns = TryGetColumn(batch, "col_name"); + var names = TryGetColumn(batch, "constraintName"); + var sequences = TryGetColumn(batch, "keySeq"); + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + if (columns == null) continue; + for (int row = 0; row < batch.Length; row++) + if (String(columns, row) is string column) + yield return new PrimaryKeyRow(String(catalogs, row) ?? catalog, + String(schemas, row) ?? schema, String(tables, row) ?? table, + column, Integer(sequences, row) ?? ++sequence, String(names, row) ?? ""); + } + } + + internal static IEnumerable ForeignKeys( + MetadataBatches result, string? parentCatalog, string? parentSchema, string? parentTable, + string catalog, string schema, string table) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, + parentCatalog: parentCatalog, parentSchema: parentSchema, parentTable: parentTable)) + if (row.String("FKCOLUMN_NAME") is string column) + yield return new ForeignKeyRow(row.String("PKTABLE_CAT") ?? parentCatalog ?? "", + row.String("PKTABLE_SCHEM") ?? parentSchema ?? "", row.String("PKTABLE_NAME") ?? parentTable ?? "", + row.String("PKCOLUMN_NAME") ?? "", row.String("FKTABLE_CAT") ?? catalog, + row.String("FKTABLE_SCHEM") ?? schema, row.String("FKTABLE_NAME") ?? table, column, + checked((int)(row.Integer("KEQ_SEQ") ?? 0)), + checked((int)(row.Integer("UPDATE_RULE") ?? 0)), + checked((int)(row.Integer("DELETE_RULE") ?? 0)), row.String("FK_NAME") ?? "", + row.String("PK_NAME"), checked((int)(row.Integer("DEFERRABILITY") ?? 5))); + yield break; + } + + int sequence = 0; + foreach (var batch in result.Batches) + { + var parentCatalogs = TryGetColumn(batch, "parentCatalogName"); + var parentSchemas = TryGetColumn(batch, "parentNamespace"); + var parentTables = TryGetColumn(batch, "parentTableName"); + var parentColumns = TryGetColumn(batch, "parentColName"); + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var columns = TryGetColumn(batch, "col_name"); + var names = TryGetColumn(batch, "constraintName"); + var sequences = TryGetColumn(batch, "keySeq"); + var updateRules = TryGetColumn(batch, "updateRule"); + var deleteRules = TryGetColumn(batch, "deleteRule"); + var deferrabilities = TryGetColumn(batch, "deferrability"); + if (columns == null) continue; + for (int row = 0; row < batch.Length; row++) + { + string? column = String(columns, row); + if (column == null) continue; + string? rowParentCatalog = String(parentCatalogs, row); + string? rowParentSchema = String(parentSchemas, row); + string? rowParentTable = String(parentTables, row); + // Match raw server identifiers before applying output fallbacks. + if (!MatchesParent(parentCatalog, parentSchema, parentTable, + rowParentCatalog, rowParentSchema, rowParentTable)) continue; + yield return new ForeignKeyRow(rowParentCatalog ?? parentCatalog ?? "", + rowParentSchema ?? parentSchema ?? "", rowParentTable ?? parentTable ?? "", + String(parentColumns, row) ?? "", String(catalogs, row) ?? catalog, + String(schemas, row) ?? schema, String(tables, row) ?? table, column, + Integer(sequences, row) ?? ++sequence, Integer(updateRules, row) ?? 0, + Integer(deleteRules, row) ?? 0, String(names, row) ?? "", null, + Integer(deferrabilities, row) ?? 5); + } + } + } internal static int CompareTables( (string? Catalog, string? Schema, string? Table, string? TableType) left, (string? Catalog, string? Schema, string? Table, string? TableType) right) { int order = StringComparer.Ordinal.Compare( - NativeMetadataResultBuilder.DefaultTableType(left.TableType), - NativeMetadataResultBuilder.DefaultTableType(right.TableType)); + DefaultTableType(left.TableType), + DefaultTableType(right.TableType)); if (order != 0) return order; order = StringComparer.Ordinal.Compare(left.Catalog, right.Catalog); if (order != 0) return order; @@ -293,12 +511,25 @@ internal static int CompareTables( return order != 0 ? order : StringComparer.Ordinal.Compare(left.Table, right.Table); } - internal static bool MatchesCatalog(string? requested, string? actual) - => requested == null || string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase); + internal static bool MatchesCatalog(CatalogFilter filter, string? actual) + => filter.Value == null || (actual != null && (filter.IsPattern + ? MatchesCatalogPattern(filter.Value, actual) + : string.Equals(filter.Value, actual, StringComparison.OrdinalIgnoreCase))); + + private static bool MatchesParent(string? catalog, string? schema, string? table, + string? actualCatalog, string? actualSchema, string? actualTable) + => MatchesCatalog(CatalogFilter.Exact(catalog), actualCatalog) && + MatchesCatalog(CatalogFilter.Exact(schema), actualSchema) && + MatchesCatalog(CatalogFilter.Exact(table), actualTable); + + internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value!; private static string? String(StringArray? values, int row) => values == null || values.IsNull(row) ? null : values.GetString(row); + private static int? Integer(Int32Array? values, int row) + => values == null || values.IsNull(row) ? null : values.GetValue(row); + private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray { try @@ -311,6 +542,29 @@ internal static bool MatchesCatalog(string? requested, string? actual) } } + private static string? LiteralCatalog(string? pattern) + { + if (pattern == null) return null; + var literal = new StringBuilder(); + bool escaped = false; + foreach (char character in pattern) + { + if (!escaped) + { + if (character is '%' or '_') return null; + if (character == '\\') + { + escaped = true; + continue; + } + } + literal.Append(character); + escaped = false; + } + if (escaped) literal.Append('\\'); + return literal.ToString(); + } + private static bool MatchesCatalogPattern(string? pattern, string catalog) { if (pattern == null) return true; diff --git a/csharp/src/StatementExecution/NativeMetadataColumns.cs b/csharp/src/StatementExecution/NativeMetadataColumns.cs index a19eb5f5..8ebc6394 100644 --- a/csharp/src/StatementExecution/NativeMetadataColumns.cs +++ b/csharp/src/StatementExecution/NativeMetadataColumns.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Linq; using AdbcDrivers.HiveServer2; using Apache.Arrow; using Apache.Arrow.Types; @@ -28,13 +29,14 @@ internal sealed class NativeMetadataColumns internal NativeMetadataColumns(RecordBatch batch, Schema schema, MetadataOperation operation) { - int count = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); + var fields = schema.FieldsList.Where(field => + operation != MetadataOperation.GetColumns || field.Name != "BASE_TYPE_NAME").ToList(); + int count = fields.Count; if (batch.ColumnCount != count) throw new DatabricksException($"Invalid native {operation} result: expected {count} columns, found {batch.ColumnCount}"); - for (int i = 0; i < count; i++) + foreach (var expected in fields) { - Field expected = schema.FieldsList[i]; int index = batch.Schema.GetFieldIndex(expected.Name); if (index < 0) throw new DatabricksException($"Invalid native {operation} result: missing {expected.Name}"); @@ -50,9 +52,25 @@ internal NativeMetadataColumns(RecordBatch batch, Schema schema, MetadataOperati } } - internal string? String(string name, int row) => NativeMetadataResultBuilder.ReadString(_columns[name], row); + internal string? String(string name, int row) + { + var array = (StringArray)_columns[name]; + return array.IsNull(row) ? null : array.GetString(row); + } - internal long? Integer(string name, int row) => NativeMetadataResultBuilder.ReadInteger(_columns[name], row); + internal long? Integer(string name, int row) + { + var array = _columns[name]; + if (array.IsNull(row)) return null; + return array switch + { + Int8Array values => values.GetValue(row), + Int16Array values => values.GetValue(row), + Int32Array values => values.GetValue(row), + Int64Array values => values.GetValue(row), + _ => throw new DatabricksException($"Expected a native metadata integer, found {array.GetType().Name}") + }; + } private static bool IsInteger(ArrowTypeId type) => type is ArrowTypeId.Int8 or ArrowTypeId.Int16 or ArrowTypeId.Int32 or ArrowTypeId.Int64; diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs index 19d337b6..72571cb8 100644 --- a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; -using System.Linq; using AdbcDrivers.HiveServer2; using AdbcDrivers.HiveServer2.Hive2; using Apache.Arrow; @@ -28,87 +27,40 @@ namespace AdbcDrivers.Databricks.StatementExecution internal static class NativeMetadataResultBuilder { internal static QueryResult Build( - IReadOnlyList batches, Schema schema, MetadataOperation operation, - string? requestedCatalog = null, IReadOnlyCollection? tableTypes = null, - string? parentCatalog = null, string? parentSchema = null, string? parentTable = null, - IReadOnlyList? sourceCatalogs = null, bool requireExactCatalog = false, - bool requireExactSourceCatalog = false) - { - // C# exposes BASE_TYPE_NAME after the 23 Thrift GetColumns fields. - int sourceColumns = schema.FieldsList.Count - (operation == MetadataOperation.GetColumns ? 1 : 0); - if (sourceCatalogs != null && sourceCatalogs.Count != batches.Count) - throw new ArgumentException("Each native batch must have a source catalog", nameof(sourceCatalogs)); - string? catalogField = operation switch - { - MetadataOperation.GetSchemas => "TABLE_CATALOG", - MetadataOperation.GetTables or MetadataOperation.GetColumns => "TABLE_CAT", - _ => null, - }; - var rows = new List<(NativeMetadataColumns Columns, int Index, string? Catalog)>(); - for (int batchIndex = 0; batchIndex < batches.Count; batchIndex++) - { - var batch = batches[batchIndex]; - var columns = new NativeMetadataColumns(batch, schema, operation); - string? sourceCatalog = sourceCatalogs?[batchIndex] ?? requestedCatalog; - - for (int row = 0; row < batch.Length; row++) - { - if (operation == MetadataOperation.GetColumns && requireExactSourceCatalog && - !MetadataRowReader.MatchesCatalog(sourceCatalog, columns.String("TABLE_CAT", row) ?? sourceCatalog)) - continue; - // Native metadata can expand wildcard characters in a quoted catalog. - if (catalogField != null && requestedCatalog != null && - (operation == MetadataOperation.GetTables || requireExactCatalog) && - !MetadataRowReader.MatchesCatalog(requestedCatalog, columns.String(catalogField, row) ?? sourceCatalog)) - continue; - if (operation == MetadataOperation.GetTables && tableTypes != null && - !tableTypes.Contains(DefaultTableType(columns.String("TABLE_TYPE", row)))) - continue; - - if (operation == MetadataOperation.GetCrossReference && - (!Matches(parentCatalog, columns.String(schema.FieldsList[0].Name, row)) || - !Matches(parentSchema, columns.String(schema.FieldsList[1].Name, row)) || - !Matches(parentTable, columns.String(schema.FieldsList[2].Name, row)))) - continue; - - rows.Add((columns, row, sourceCatalog)); - } - } - - if (operation == MetadataOperation.GetTables) - { - rows.Sort((left, right) => MetadataRowReader.CompareTables( - (left.Columns.String("TABLE_CAT", left.Index) ?? left.Catalog, - left.Columns.String("TABLE_SCHEM", left.Index), - left.Columns.String("TABLE_NAME", left.Index), - left.Columns.String("TABLE_TYPE", left.Index)), - (right.Columns.String("TABLE_CAT", right.Index) ?? right.Catalog, - right.Columns.String("TABLE_SCHEM", right.Index), - right.Columns.String("TABLE_NAME", right.Index), - right.Columns.String("TABLE_TYPE", right.Index)))); - } + MetadataBatches result, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + => Build(new[] { result }, schema, operation, tableTypes, parentCatalog, parentSchema, parentTable); + internal static QueryResult Build( + IReadOnlyList results, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + { + var rows = MetadataRowReader.NativeRows( + results, schema, operation, tableTypes, parentCatalog, parentSchema, parentTable); var arrays = new List(schema.FieldsList.Count); - for (int column = 0; column < schema.FieldsList.Count; column++) + foreach (var field in schema.FieldsList) { - switch (schema.FieldsList[column].DataType.TypeId) + switch (field.DataType.TypeId) { case ArrowTypeId.String: var strings = new StringArray.Builder(); - foreach (var (columns, row, sourceCatalog) in rows) + foreach (var row in rows) { - string? typeName = column == sourceColumns && operation == MetadataOperation.GetColumns - ? columns.String("TYPE_NAME", row) - : null; - string? value = column == sourceColumns && operation == MetadataOperation.GetColumns - ? typeName == null ? null : ColumnMetadataHelper.GetBaseTypeName(typeName) - : columns.String(schema.FieldsList[column].Name, row); - if (column == 1 && operation == MetadataOperation.GetSchemas) - value ??= sourceCatalog ?? ""; - if (column == 0 && (operation == MetadataOperation.GetTables || operation == MetadataOperation.GetColumns)) - value ??= sourceCatalog ?? ""; - if (column == 3 && operation == MetadataOperation.GetTables) - value = DefaultTableType(value); + string? value; + if (operation == MetadataOperation.GetColumns && field.Name == "BASE_TYPE_NAME") + { + string? typeName = row.String("TYPE_NAME"); + value = typeName == null ? null : ColumnMetadataHelper.GetBaseTypeName(typeName); + } + else + value = row.String(field.Name); + if ((field.Name == "TABLE_CATALOG" && operation == MetadataOperation.GetSchemas) || + (field.Name == "TABLE_CAT" && operation is MetadataOperation.GetTables or MetadataOperation.GetColumns)) + value ??= row.SourceCatalog ?? ""; + if (field.Name == "TABLE_TYPE" && operation == MetadataOperation.GetTables) + value = MetadataRowReader.DefaultTableType(value); if (value == null) strings.AppendNull(); else strings.Append(value); } arrays.Add(strings.Build()); @@ -116,94 +68,55 @@ internal static QueryResult Build( case ArrowTypeId.Int8: var int8 = new Int8Array.Builder(); - foreach (var (columns, row, _) in rows) - { - long? value = columns.Integer(schema.FieldsList[column].Name, row); - if (value.HasValue) int8.Append(checked((sbyte)value.Value)); else int8.AppendNull(); - } - arrays.Add(int8.Build()); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int8.Append(value.HasValue ? checked((sbyte)value.Value) : (sbyte?)null), + () => int8.Build())); break; case ArrowTypeId.Int16: var int16 = new Int16Array.Builder(); - foreach (var (columns, row, _) in rows) - { - long? value = columns.Integer(schema.FieldsList[column].Name, row); - if (value.HasValue) int16.Append(checked((short)value.Value)); else int16.AppendNull(); - } - arrays.Add(int16.Build()); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int16.Append(value.HasValue ? checked((short)value.Value) : (short?)null), + () => int16.Build())); break; case ArrowTypeId.Int32: var int32 = new Int32Array.Builder(); - foreach (var (columns, row, _) in rows) - { - string name = schema.FieldsList[column].Name; - long? value = columns.Integer(name, row); - if (operation == MetadataOperation.GetColumns && (name is "COLUMN_SIZE" or "DECIMAL_DIGITS")) - { - // Thrift defaults null sizes and digits to zero before type-specific overrides. - value ??= 0; - string? typeName = columns.String("TYPE_NAME", row); - if (typeName != null) - { - string baseName = ColumnMetadataHelper.GetBaseTypeName(typeName); - if (baseName is "DECIMAL" or "NUMERIC") - value = name == "COLUMN_SIZE" - ? ColumnMetadataHelper.GetColumnSizeDefault(typeName) - : ColumnMetadataHelper.GetDecimalDigitsDefault(typeName); - else if (name == "COLUMN_SIZE" && baseName is - "CHAR" or "NCHAR" or "VARCHAR" or "NVARCHAR" or "LONGVARCHAR" or "LONGNVARCHAR" or "STRING") - value = ColumnMetadataHelper.GetColumnSizeDefault(typeName); - } - } - if (value.HasValue) int32.Append(checked((int)value.Value)); else int32.AppendNull(); - } - arrays.Add(int32.Build()); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int32.Append(value.HasValue ? checked((int)value.Value) : (int?)null), + () => int32.Build())); break; case ArrowTypeId.Int64: var int64 = new Int64Array.Builder(); - foreach (var (columns, row, _) in rows) - { - long? value = columns.Integer(schema.FieldsList[column].Name, row); - if (value.HasValue) int64.Append(value.Value); else int64.AppendNull(); - } - arrays.Add(int64.Build()); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int64.Append(value), () => int64.Build())); break; default: - throw new DatabricksException($"Unsupported native metadata field type: {schema.FieldsList[column].DataType}"); + throw new DatabricksException($"Unsupported native metadata field type: {field.DataType}"); } } return new QueryResult(rows.Count, new HiveInfoArrowStream(schema, arrays.ToArray())); } - internal static string? ReadString(IArrowArray array, int row) + private static IArrowArray BuildIntegers( + IReadOnlyList rows, string field, MetadataOperation operation, + Action append, Func build) { - if (array.IsNull(row)) return null; - return array is StringArray strings - ? strings.GetString(row) - : throw new DatabricksException($"Expected a native metadata string, found {array.GetType().Name}"); - } - - internal static long? ReadInteger(IArrowArray array, int row) - { - if (array.IsNull(row)) return null; - return array switch + foreach (var row in rows) { - Int8Array values => values.GetValue(row), - Int16Array values => values.GetValue(row), - Int32Array values => values.GetValue(row), - Int64Array values => values.GetValue(row), - _ => throw new DatabricksException($"Expected a native metadata integer, found {array.GetType().Name}") - }; + long? value = row.Integer(field); + if (operation == MetadataOperation.GetColumns && (field is "COLUMN_SIZE" or "DECIMAL_DIGITS")) + { + var precisionAndScale = ColumnMetadataHelper.NormalizePrecisionAndScale( + row.String("TYPE_NAME"), row.Integer("COLUMN_SIZE"), row.Integer("DECIMAL_DIGITS")); + value = field == "COLUMN_SIZE" ? precisionAndScale.ColumnSize : precisionAndScale.DecimalDigits; + } + append(value); + } + return build(); } - - private static bool Matches(string? requested, string? actual) - => requested == null || (actual != null && string.Equals(requested, actual, StringComparison.OrdinalIgnoreCase)); - - internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value!; } } diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index 4eae36a2..4add31a4 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net.Http; using System.Threading; @@ -717,7 +718,8 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string try { columns = ReadColumnsAsync( - resolvedCatalog, dbSchema, tableName, null, cts.Token, requireExactCatalog: true) + resolvedCatalog, dbSchema, tableName, null, cts.Token, + catalogFilter: CatalogFilter.Exact(resolvedCatalog)) .GetAwaiter().GetResult(); } catch (Exception ex) when (cts.IsCancellationRequested && ex is not TimeoutException) @@ -746,37 +748,20 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string async Task> IGetObjectsDataProvider.GetCatalogsAsync(string? catalogPattern, CancellationToken cancellationToken) { string sql = new ShowCatalogsCommand(catalogPattern).Build(); - MetadataBatches result; - try - { - result = await ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - // Object-not-found → empty (GetObjects nested-shape path; matches the - // flat metadata methods and the JDBC reference driver). - return System.Array.Empty(); - } - return MetadataRowReader.Catalogs(result, catalogPattern).ToList(); + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetCatalogs, cancellationToken, + catalogFilter: CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Catalogs(result).ToList(); } async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) { string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); - MetadataBatches result; - try - { - result = await ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return System.Array.Empty<(string, string)>(); - } - - return MetadataRowReader.Schemas(result, catalogPattern) + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetSchemas, cancellationToken, catalogPattern, + CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Schemas(result) .Select(row => (row.Catalog, row.Schema)).ToList(); } @@ -795,17 +780,10 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); - MetadataBatches result; - try - { - result = await ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return System.Array.Empty<(string, string, string, string)>(); - } - return MetadataRowReader.Tables(result, catalogPattern, tableTypes) + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetTables, cancellationToken, catalogPattern, + CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Tables(result, tableTypes) .Select(row => (row.Catalog, row.Schema, row.Table, row.TableType)).ToList(); } @@ -814,31 +792,18 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - ColumnMetadataResult columns; - try - { - columns = await ReadColumnsAsync(catalogPattern, schemaPattern, tablePattern, - columnPattern, cancellationToken, requireExactCatalog: true).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return; - } - - var tablePositions = new Dictionary(); + var columns = await ReadMetadataOrEmptyAsync(MetadataOperation.GetColumns, + () => ReadColumnsAsync(catalogPattern, schemaPattern, tablePattern, + columnPattern, cancellationToken, CatalogFilter.Pattern(catalogPattern), + requireTableIdentifiers: true), + () => new ColumnMetadataResult(System.Array.Empty())).ConfigureAwait(false); foreach (var column in columns.Rows) { - if (column.Catalog == null || column.Schema == null || column.Table == null || - string.IsNullOrEmpty(column.Name)) continue; - string tableKey = $"{column.Catalog}.{column.Schema}.{column.Table}"; - tablePositions.TryGetValue(tableKey, out int position); - tablePositions[tableKey] = ++position; - int ordinal = column.IsNative ? checked(column.Ordinal + 1) : position; - if (catalogMap.TryGetValue(column.Catalog, out var schemaMap) - && schemaMap.TryGetValue(column.Schema, out var tableMap) - && tableMap.TryGetValue(column.Table, out var tableInfo)) + if (catalogMap.TryGetValue(column.Catalog!, out var schemaMap) + && schemaMap.TryGetValue(column.Schema!, out var tableMap) + && tableMap.TryGetValue(column.Table!, out var tableInfo)) ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfo, column.Name, column.TypeName, ordinal, column.Nullable, + tableInfo, column.Name, column.TypeName, checked(column.Ordinal + 1), column.Nullable, columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); } } @@ -850,7 +815,8 @@ internal async Task> ExecuteMetadataSqlAsync(string sql, Cance } internal async Task ExecuteMetadataCommandAsync( - string sql, MetadataOperation? operation, CancellationToken cancellationToken) + string sql, MetadataOperation? operation, CancellationToken cancellationToken, + string? sourceCatalog = null, CatalogFilter catalogFilter = default) { var batches = new List(); using var stmt = (StatementExecutionStatement)CreateStatement(); @@ -858,7 +824,7 @@ internal async Task ExecuteMetadataCommandAsync( var result = await stmt.ExecuteQueryAsync( cancellationToken, isMetadataExecution: true, metadataOperation: operation).ConfigureAwait(false); using var stream = result.Stream; - if (stream == null) return new MetadataBatches(batches, stmt.IsNativeMetadataResult); + if (stream == null) return new MetadataBatches(batches, stmt.IsNativeMetadataResult, sourceCatalog, catalogFilter); while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -866,7 +832,41 @@ internal async Task ExecuteMetadataCommandAsync( if (batch == null) break; batches.Add(batch); } - return new MetadataBatches(batches, stmt.IsNativeMetadataResult); + return new MetadataBatches(batches, stmt.IsNativeMetadataResult, sourceCatalog, catalogFilter); + } + + internal Task ReadMetadataAsync( + string sql, MetadataOperation operation, CancellationToken cancellationToken, + string? sourceCatalog = null, CatalogFilter catalogFilter = default) + { + Activity.Current?.SetTag("sql_query", sql); + return ReadMetadataOrEmptyAsync(operation, + () => ExecuteMetadataCommandAsync(sql, operation, cancellationToken, sourceCatalog, catalogFilter), + () => new MetadataBatches(new List(), false, sourceCatalog, catalogFilter)); + } + + internal static async Task ReadMetadataOrEmptyAsync( + MetadataOperation operation, Func> read, Func empty) + { + try + { + return await read().ConfigureAwait(false); + } + catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) + { + string name = operation switch + { + MetadataOperation.GetCatalogs => "get_catalogs", + MetadataOperation.GetSchemas => "get_schemas", + MetadataOperation.GetTables => "get_tables", + MetadataOperation.GetColumns => "get_columns", + MetadataOperation.GetPrimaryKeys => "get_primary_keys", + MetadataOperation.GetCrossReference => "get_cross_reference", + _ => operation.ToString(), + }; + Activity.Current?.AddEvent($"statement.{name}.object_not_found", [new("error", ex.Message)]); + return empty(); + } } internal List ExecuteMetadataSql(string sql, CancellationToken cancellationToken = default) @@ -877,16 +877,17 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc // SHOW COLUMNS IN ALL CATALOGS is unavailable, so fan out over catalogs. internal async Task ReadColumnsAsync( string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, - CancellationToken cancellationToken, bool requireExactCatalog = false) + CancellationToken cancellationToken, CatalogFilter catalogFilter = default, + bool requireTableIdentifiers = false) { var results = new List(); if (catalog != null) { string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); var result = await ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); - results.Add(new MetadataBatches(result.Batches, result.IsNative, catalog, requireExactCatalog)); - return new ColumnMetadataResult(results); + sql, MetadataOperation.GetColumns, cancellationToken, catalog, catalogFilter).ConfigureAwait(false); + results.Add(result); + return new ColumnMetadataResult(results, requireTableIdentifiers); } string catalogsSql = new ShowCatalogsCommand(null).Build(); @@ -900,17 +901,17 @@ internal async Task ReadColumnsAsync( try { result = await ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetColumns, cancellationToken).ConfigureAwait(false); + sql, MetadataOperation.GetColumns, cancellationToken, + sourceCatalog, CatalogFilter.Exact(sourceCatalog)).ConfigureAwait(false); } catch { // Skip catalogs we can't access (permission errors) continue; } - results.Add(new MetadataBatches(result.Batches, result.IsNative, - sourceCatalog, requireExactCatalog: true)); + results.Add(result); } - return new ColumnMetadataResult(results); + return new ColumnMetadataResult(results, requireTableIdentifiers); } internal bool EnablePKFK => _enablePKFK; diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index c688a767..4cea7bcb 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1298,7 +1298,7 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT IsNativeMetadataResult = result.IsNative; if (result.IsNative) return NativeMetadataResultBuilder.Build( - result.Batches, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); + result, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); var tableCatBuilder = new StringArray.Builder(); int count = 0; @@ -1333,34 +1333,18 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo EscapePatternWildcardsInName(_metadataSchemaName)).Build(); activity?.SetTag("sql_query", sql); - // Object-not-found (missing catalog/schema/table, or the server rejecting - // an empty/invalid name) → return an EMPTY result, matching both the Thrift - // path and the JDBC reference driver (isObjectNotFoundException). Restores - // the #388 behavior; the "make SEA throw" premise was disproven — Thrift - // returns empty on object-not-found too. - MetadataBatches result; - try - { - result = await _connection.ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetSchemas, cancellationToken).ConfigureAwait(false); - IsNativeMetadataResult = result.IsNative; - if (result.IsNative) - return NativeMetadataResultBuilder.Build( - result.Batches, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas, - requestedCatalog: catalog, requireExactCatalog: _escapePatternWildcards); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_schemas.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptySchemasResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetSchemas, cancellationToken, catalog, + CatalogFilter.Exact(_escapePatternWildcards ? catalog : null)).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); var tableSchemaBuilder = new StringArray.Builder(); var tableCatalogBuilder = new StringArray.Builder(); int count = 0; - foreach (var row in MetadataRowReader.Schemas(result, catalog)) + foreach (var row in MetadataRowReader.Schemas(result)) { tableSchemaBuilder.Append(row.Schema); tableCatalogBuilder.Append(row.Catalog); @@ -1403,38 +1387,11 @@ private async Task GetTablesAsync(CancellationToken cancellationTok EscapePatternWildcardsInName(_metadataTableName)).Build(); activity?.SetTag("sql_query", sql); - MetadataBatches result; - try - { - result = await _connection.ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetTables, cancellationToken).ConfigureAwait(false); - IsNativeMetadataResult = result.IsNative; - if (result.IsNative) - { - var requestedTypes = _metadataTableTypes?.Split(',').Select(type => type.Trim()).ToArray(); - return NativeMetadataResultBuilder.Build( - result.Batches, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, - requestedCatalog: catalog, tableTypes: requestedTypes); - } - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_tables.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptyTablesResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetTables, cancellationToken, catalog, + CatalogFilter.Exact(catalog)).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; - var tableCatBuilder = new StringArray.Builder(); - var tableSchemaBuilder = new StringArray.Builder(); - var tableNameBuilder = new StringArray.Builder(); - var tableTypeBuilder = new StringArray.Builder(); - var remarksBuilder = new StringArray.Builder(); - var typeCatBuilder = new StringArray.Builder(); - var typeSchemaBuilder = new StringArray.Builder(); - var typeNameBuilder = new StringArray.Builder(); - var selfRefColBuilder = new StringArray.Builder(); - var refGenBuilder = new StringArray.Builder(); // Issue #526: match JDBC's MetadataResultSetBuilder and the Thrift path - // the types filter is a case-SENSITIVE exact match against the server's // uppercase type names (TABLE/VIEW/...). Use Ordinal, not OrdinalIgnoreCase. @@ -1446,9 +1403,24 @@ private async Task GetTablesAsync(CancellationToken cancellationTok StringComparer.Ordinal) : null; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, + tableTypes: tableTypeFilter); + + var tableCatBuilder = new StringArray.Builder(); + var tableSchemaBuilder = new StringArray.Builder(); + var tableNameBuilder = new StringArray.Builder(); + var tableTypeBuilder = new StringArray.Builder(); + var remarksBuilder = new StringArray.Builder(); + var typeCatBuilder = new StringArray.Builder(); + var typeSchemaBuilder = new StringArray.Builder(); + var typeNameBuilder = new StringArray.Builder(); + var selfRefColBuilder = new StringArray.Builder(); + var refGenBuilder = new StringArray.Builder(); int count = 0; foreach (var row in MetadataRowReader.Tables( - result, catalog, tableTypeFilter, normalizeEmptyTableType: false)) + result, tableTypeFilter, normalizeEmptyTableType: false)) { tableCatBuilder.Append(row.Catalog); tableSchemaBuilder.Append(row.Schema); @@ -1494,35 +1466,22 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - ColumnMetadataResult columns; - try - { - columns = await _connection.ReadColumnsAsync( + var columns = await StatementExecutionConnection.ReadMetadataOrEmptyAsync( + MetadataOperation.GetColumns, + () => _connection.ReadColumnsAsync( catalog, EscapePatternWildcardsInName(_metadataSchemaName), EscapePatternWildcardsInName(_metadataTableName), EscapePatternWildcardsInName(_metadataColumnName), - cancellationToken, requireExactCatalog: _escapePatternWildcards).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_columns.object_not_found", [ - new("error", ex.Message) - ]); - return FlatColumnsResultBuilder.BuildFlatColumnsResult( - System.Array.Empty<(string, string, string, TableInfo)>()); - } + cancellationToken, CatalogFilter.Exact(_escapePatternWildcards ? catalog : null)), + () => new ColumnMetadataResult(System.Array.Empty())).ConfigureAwait(false); IsNativeMetadataResult = columns.IsNative; if (IsNativeMetadataResult) { return NativeMetadataResultBuilder.Build( - columns.Batches, - MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, - requestedCatalog: catalog, - sourceCatalogs: columns.SourceCatalogs, - requireExactCatalog: _escapePatternWildcards, - requireExactSourceCatalog: catalog == null); + columns.Results, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); } var tableInfos = new Dictionary(); @@ -1776,57 +1735,15 @@ private async Task GetPrimaryKeysAsyncNoThrow( string sql = new ShowKeysCommand(catalog!, schema!, table!).Build(); activity?.SetTag("sql_query", sql); - List batches; - try - { - var metadataResult = await _connection.ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); - batches = metadataResult.Batches; - IsNativeMetadataResult = metadataResult.IsNative; - if (metadataResult.IsNative) - { - return NativeMetadataResultBuilder.Build( - batches, MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys); - } - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_primary_keys.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys); - var keys = new List<(string, string, string, string, int, string)>(); - int seq = 0; - foreach (var batch in batches) - { - var colNameArray = TryGetColumn(batch, "col_name"); - var keyNameArray = TryGetColumn(batch, "constraintName"); - var keySeqArray = TryGetColumn(batch, "keySeq"); - // Read the identifier columns back from the SHOW KEYS response so - // TABLE_CAT/TABLE_SCHEM/TABLE_NAME reflect the server's stored (canonical) - // casing rather than echoing the caller's input case. Mirrors the JDBC - // reference driver (PRIMARY_KEYS_COLUMNS maps TABLE_CAT→catalogName, - // TABLE_SCHEM→namespace, TABLE_NAME→tableName) and this driver's own - // FetchCrossReferenceAsync. Fall back to the input args when the server - // omits a value, so behavior is never worse than before. - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - if (colNameArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (colNameArray.IsNull(i)) continue; - int keySeq = keySeqArray != null && !keySeqArray.IsNull(i) ? keySeqArray.GetValue(i)!.Value : ++seq; - string pkName = keyNameArray != null && !keyNameArray.IsNull(i) ? keyNameArray.GetString(i) : ""; - string rowCatalog = catalogArray != null && !catalogArray.IsNull(i) ? catalogArray.GetString(i) : catalog!; - string rowSchema = schemaArray != null && !schemaArray.IsNull(i) ? schemaArray.GetString(i) : schema!; - string rowTable = tableArray != null && !tableArray.IsNull(i) ? tableArray.GetString(i) : table!; - keys.Add((rowCatalog, rowSchema, rowTable, - colNameArray.GetString(i), keySeq, pkName)); - } - } + var keys = MetadataRowReader.PrimaryKeys(result, catalog!, schema!, table!) + .Select(row => (row.Catalog, row.Schema, row.Table, row.Column, row.Sequence, row.Name)).ToList(); activity?.SetTag("result_count", keys.Count); return MetadataSchemaFactory.BuildPrimaryKeysResult(keys); @@ -1859,7 +1776,7 @@ private async Task GetCrossReferenceAsync(CancellationToken cancell // Pass the parent catalog to the filter ONLY when the caller explicitly supplied it. When // unset, _metadataCatalogName holds the connection's seeded default catalog; using it as the - // parent-catalog filter (added in the ParentMatches step) wrongly drops every FK whose real + // parent-catalog filter wrongly drops every FK whose real // parent is in another catalog — e.g. getImportedKeys (no parent supplied) returns empty while // Thrift returns the row. Parent schema/table are not seeded, so they already no-op when unset // and are passed through as-is; only the catalog needs this guard. @@ -1906,120 +1823,25 @@ private async Task GetCrossReferenceAsyncNoThrow( string sql = new ShowForeignKeysCommand(fkCatalog!, fkSchema!, fkTable!).Build(); activity?.SetTag("sql_query", sql); - List batches; - try - { - var metadataResult = await _connection.ExecuteMetadataCommandAsync( - sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); - batches = metadataResult.Batches; - IsNativeMetadataResult = metadataResult.IsNative; - if (metadataResult.IsNative) - { - return NativeMetadataResultBuilder.Build( - batches, MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, - parentCatalog: pkCatalog, parentSchema: pkSchema, parentTable: pkTable); - } - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); - } - - var refs = new List<(string, string, string, string, string, string, string, string, int, int, int, string, string?, int)>(); - int seq = 0; - foreach (var batch in batches) - { - var pkCatalogArray = TryGetColumn(batch, "parentCatalogName"); - var pkSchemaArray = TryGetColumn(batch, "parentNamespace"); - var pkTableArray = TryGetColumn(batch, "parentTableName"); - var pkColArray = TryGetColumn(batch, "parentColName"); - var fkCatalogArray = TryGetColumn(batch, "catalogName"); - var fkSchemaArray = TryGetColumn(batch, "namespace"); - var fkTableArray = TryGetColumn(batch, "tableName"); - var fkColArray = TryGetColumn(batch, "col_name"); - var fkNameArray = TryGetColumn(batch, "constraintName"); - var fkKeySeqArray = TryGetColumn(batch, "keySeq"); - var fkUpdateRuleArray = TryGetColumn(batch, "updateRule"); - var fkDeleteRuleArray = TryGetColumn(batch, "deleteRule"); - var fkDeferrabilityArray = TryGetColumn(batch, "deferrability"); - - if (fkColArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (fkColArray.IsNull(i)) continue; - - // Raw server-provided parent identifiers, null when the column is - // absent or NULL for this row. Filtering compares against these raw - // values (not the value-population fallback below) so a requested - // parent can never match a null server column against itself. - string? srvPkCatalog = pkCatalogArray != null && !pkCatalogArray.IsNull(i) ? pkCatalogArray.GetString(i) : null; - string? srvPkSchema = pkSchemaArray != null && !pkSchemaArray.IsNull(i) ? pkSchemaArray.GetString(i) : null; - string? srvPkTable = pkTableArray != null && !pkTableArray.IsNull(i) ? pkTableArray.GetString(i) : null; - - // SHOW FOREIGN KEYS is scoped to the FOREIGN table only, so it returns - // FKs to every parent. When the caller specified parent identifiers, - // filter the rows down to that parent — mirroring the JDBC reference - // driver (CrossReferenceKeysDatabricksResultSetAdapter.includeRow), which - // keeps a row only when its parent catalog/schema/table equalsIgnoreCase - // the requested one. A null parent arg means "no parent constraint" (the - // GetColumnsExtended foreign-only reuse passes null on all three), so it - // is NOT a filter; a specified parent that the server row doesn't match - // (including a null/empty server value) filters the row out. - if (!ParentMatches(pkCatalog, srvPkCatalog) - || !ParentMatches(pkSchema, srvPkSchema) - || !ParentMatches(pkTable, srvPkTable)) - { - continue; - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, + parentCatalog: pkCatalog, parentSchema: pkSchema, parentTable: pkTable); - // Value population: fall back to the requested parent value when the - // server column is null so the emitted row still carries an identifier. - var rowPkCatalog = srvPkCatalog ?? pkCatalog ?? ""; - var rowPkSchema = srvPkSchema ?? pkSchema ?? ""; - var rowPkTable = srvPkTable ?? pkTable ?? ""; - - refs.Add(( - rowPkCatalog, - rowPkSchema, - rowPkTable, - pkColArray != null && !pkColArray.IsNull(i) ? pkColArray.GetString(i) : "", - fkCatalogArray != null && !fkCatalogArray.IsNull(i) ? fkCatalogArray.GetString(i) : fkCatalog!, - fkSchemaArray != null && !fkSchemaArray.IsNull(i) ? fkSchemaArray.GetString(i) : fkSchema!, - fkTableArray != null && !fkTableArray.IsNull(i) ? fkTableArray.GetString(i) : fkTable!, - fkColArray.GetString(i), - fkKeySeqArray != null && !fkKeySeqArray.IsNull(i) ? fkKeySeqArray.GetValue(i)!.Value : ++seq, - fkUpdateRuleArray != null && !fkUpdateRuleArray.IsNull(i) ? fkUpdateRuleArray.GetValue(i)!.Value : 0, - fkDeleteRuleArray != null && !fkDeleteRuleArray.IsNull(i) ? fkDeleteRuleArray.GetValue(i)!.Value : 0, - fkNameArray != null && !fkNameArray.IsNull(i) ? fkNameArray.GetString(i) : "", - (string?)null, - fkDeferrabilityArray != null && !fkDeferrabilityArray.IsNull(i) ? fkDeferrabilityArray.GetValue(i)!.Value : 5 - )); - } - } + var refs = MetadataRowReader.ForeignKeys(result, pkCatalog, pkSchema, pkTable, + fkCatalog!, fkSchema!, fkTable!) + .Select(row => (row.ParentCatalog, row.ParentSchema, row.ParentTable, row.ParentColumn, + row.Catalog, row.Schema, row.Table, row.Column, row.Sequence, row.UpdateRule, + row.DeleteRule, row.Name, row.ParentName, row.Deferrability)).ToList(); activity?.SetTag("result_count", refs.Count); return MetadataSchemaFactory.BuildCrossReferenceResult(refs); }, "GetCrossReference").ConfigureAwait(false); } - private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray - { - try { return batch.Column(name) as T; } - catch (ArgumentOutOfRangeException) { return null; } - } - - /// - /// Cross-reference parent-identifier filter. A null means - /// "no parent constraint" (matches any row); a specified value matches case-insensitively, - /// mirroring the JDBC reference driver's CrossReferenceKeysDatabricksResultSetAdapter. - /// An empty-string requested value therefore matches only an empty-string row value — - /// which no real FK has — so it correctly filters everything out. A null row value - /// (server column absent/NULL) never matches a specified parent. - /// - private static bool ParentMatches(string? requested, string? rowValue) - => requested == null || (rowValue != null && string.Equals(requested, rowValue, StringComparison.OrdinalIgnoreCase)); - // TracingStatement implementation public override string AssemblyVersion => GetType().Assembly.GetName().Version?.ToString() ?? "1.0.0"; public override string AssemblyName => "AdbcDrivers.Databricks"; diff --git a/csharp/test/ColumnMetadataHelperTests.cs b/csharp/test/ColumnMetadataHelperTests.cs index e4c3610c..55e509f1 100644 --- a/csharp/test/ColumnMetadataHelperTests.cs +++ b/csharp/test/ColumnMetadataHelperTests.cs @@ -15,6 +15,7 @@ */ using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2.Hive2; using Xunit; using static AdbcDrivers.HiveServer2.Hive2.HiveServer2Connection; @@ -202,5 +203,20 @@ public void GetDataTypeCode_HandlesTrimAndCase(string typeName, short expectedCo Assert.Equal(expectedCode, ColumnMetadataHelper.GetDataTypeCode(typeName)); } + [Theory] + [InlineData("DECIMAL(10,2)", 10, 2)] + [InlineData("DOUBLE", 8, 15)] + [InlineData("VARCHAR(42)", 42, 0)] + [InlineData("BINARY", 0, 0)] + public void PopulateTableInfo_PreservesComputedPrecisionAndScale(string typeName, int size, int digits) + { + var tableInfo = new TableInfo("TABLE"); + + ColumnMetadataHelper.PopulateTableInfoFromTypeName(tableInfo, "a", typeName, 0); + + Assert.Equal(size, Assert.Single(tableInfo.Precision)); + Assert.Equal((short)digits, Assert.Single(tableInfo.Scale)); + } + } } diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs index 9b500098..886fec05 100644 --- a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -34,12 +34,13 @@ public void Catalogs_OnlyFilterNativeResponses(bool native) { using var batch = Strings((native ? "TABLE_CAT" : "catalog", new string?[] { "ma_catalog", "maxcatalog", "other" })); - var result = new MetadataBatches(new List { batch }, native); + var result = new MetadataBatches(new List { batch }, native, + catalogFilter: CatalogFilter.Pattern(@"ma\_%")); string[] expected = native ? new[] { "ma_catalog" } : new[] { "ma_catalog", "maxcatalog", "other" }; - Assert.Equal(expected, MetadataRowReader.Catalogs(result, @"ma\_%")); + Assert.Equal(expected, MetadataRowReader.Catalogs(result)); } [Theory] @@ -55,10 +56,10 @@ public void ShowTables_EmptyTypeKeepsExistingCallSiteBehavior(bool normalizeEmpt var result = new MetadataBatches(new List { batch }, false); var row = Assert.Single(MetadataRowReader.Tables( - result, "main", normalizeEmptyTableType: normalizeEmptyTableType)); + result, normalizeEmptyTableType: normalizeEmptyTableType)); Assert.Equal(expectedType, row.TableType); var filtered = MetadataRowReader.Tables( - result, "main", new[] { "TABLE" }, normalizeEmptyTableType: normalizeEmptyTableType); + result, new[] { "TABLE" }, normalizeEmptyTableType: normalizeEmptyTableType); Assert.Equal(normalizeEmptyTableType ? 1 : 0, filtered.Count); } @@ -98,30 +99,32 @@ public void ShowColumns_PartialIdentifiersStillProvideTableSchemaFields() Assert.False(row.Nullable); } - [Fact] - public async Task NativeColumns_FilterExactCatalogAndFillMissingCatalog() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NativeColumns_ShareCatalogFilterAndConcreteFallback(bool pattern) { using var batch = NativeColumns("foo_bar", "fooxbar", null); var result = new ColumnMetadataResult(new[] { - new MetadataBatches(new List { batch }, true, "foo_bar", requireExactCatalog: true), + new MetadataBatches(new List { batch }, true, "foo_bar", + pattern ? CatalogFilter.Pattern("foo_bar") : CatalogFilter.Exact("foo_bar")), }); Assert.True(result.IsNative); - Assert.Equal(3, Assert.Single(result.Batches).Length); - Assert.Equal(2, result.Rows.Count); + Assert.Equal(3, Assert.Single(Assert.Single(result.Results).Batches).Length); + Assert.Equal(pattern ? new[] { "foo_bar", "fooxbar" } : new[] { "foo_bar", "foo_bar" }, + result.Rows.Select(row => row.Catalog)); Assert.All(result.Rows, row => { - Assert.Equal("foo_bar", row.Catalog); Assert.Equal(7, row.Ordinal); Assert.Equal("7", row.Default); Assert.True(row.Nullable); Assert.True(row.IsAutoIncrement); }); - var flat = NativeMetadataResultBuilder.Build(result.Batches, - MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, - sourceCatalogs: result.SourceCatalogs, requireExactSourceCatalog: true); + var flat = NativeMetadataResultBuilder.Build(result.Results, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); using var stream = flat.Stream!; using var output = (await stream.ReadNextRecordBatchAsync())!; Assert.Equal(result.Rows.Count, output.Length); @@ -130,6 +133,68 @@ public async Task NativeColumns_FilterExactCatalogAndFillMissingCatalog() Enumerable.Range(0, output.Length).Select(row => catalogs.GetString(row))); } + public static IEnumerable NativeCatalogFallbacks() + { + foreach (var operation in new[] + { + MetadataOperation.GetSchemas, MetadataOperation.GetTables, MetadataOperation.GetColumns, + }) + foreach (var (pattern, scope, expected) in new[] + { + (false, "foo_bar", "foo_bar"), + (true, "main", "main"), + (true, @"foo\_bar", "foo_bar"), + (true, @"foo\%bar", "foo%bar"), + (true, "ma%", (string?)null), + (true, "foo_bar", (string?)null), + }) + yield return new object?[] { operation.ToString(), pattern, scope, expected }; + } + + [Theory] + [MemberData(nameof(NativeCatalogFallbacks))] + public async Task MissingNativeCatalog_OnlyUsesConcreteScope( + string operationName, bool pattern, string scope, string? expectedCatalog) + { + var operation = (MetadataOperation)System.Enum.Parse(typeof(MetadataOperation), operationName); + using var batch = operation switch + { + MetadataOperation.GetSchemas => Strings( + ("TABLE_SCHEM", new string?[] { "default" }), + ("TABLE_CATALOG", new string?[] { null })), + MetadataOperation.GetTables => Strings(MetadataSchemaFactory.CreateTablesSchema().FieldsList + .Select(field => (field.Name, new string?[] + { + field.Name switch + { + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "TABLE_TYPE" => "TABLE", + _ => null, + }, + })).ToArray()), + _ => NativeColumns(new string?[] { null }), + }; + var result = new MetadataBatches(new List { batch }, true, scope, + pattern ? CatalogFilter.Pattern(scope) : CatalogFilter.Exact(scope)); + var catalogs = operation switch + { + MetadataOperation.GetSchemas => MetadataRowReader.Schemas(result).Select(row => row.Catalog), + MetadataOperation.GetTables => MetadataRowReader.Tables(result).Select(row => row.Catalog), + _ => MetadataRowReader.Columns(new[] { result }).Select(row => row.Catalog), + }; + var expected = expectedCatalog == null ? System.Array.Empty() : new[] { expectedCatalog }; + Assert.Equal(expected, catalogs); + + var flat = NativeMetadataResultBuilder.Build(result, batch.Schema, operation); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(expected.Length, output.Length); + if (expectedCatalog != null) + Assert.Equal(expectedCatalog, ((StringArray)output.Column( + operation == MetadataOperation.GetSchemas ? "TABLE_CATALOG" : "TABLE_CAT")).GetString(0)); + } + [Fact] public void MixedColumns_PreserveServerOrdinalsAndCountShowRows() { @@ -146,6 +211,112 @@ public void MixedColumns_PreserveServerOrdinalsAndCountShowRows() Assert.Equal(new[] { "a", "b" }, result.Rows.Select(row => row.Name)); } + [Fact] + public void ShowPrimaryKeys_PreserveCanonicalNamesAndSequenceFallbackAcrossBatches() + { + using var firstStrings = Strings( + ("col_name", new string?[] { "a", "b", null }), + ("catalogName", new string?[] { "Main", null, null }), + ("namespace", new string?[] { "Default", null, null }), + ("tableName", new string?[] { "Table", null, null }), + ("constraintName", new string?[] { "pk", null, null })); + using var sequences = new Int32Array.Builder().Append(7).AppendNull().AppendNull().Build(); + using var first = new RecordBatch( + new Schema(firstStrings.Schema.FieldsList.Concat(new[] { new Field("keySeq", Int32Type.Default, true) }), null), + Enumerable.Range(0, firstStrings.ColumnCount).Select(firstStrings.Column).Concat(new[] { sequences }).ToArray(), 3); + using var second = Strings(("col_name", new string?[] { "c" })); + var result = new MetadataBatches(new List { first, second }, false); + + var rows = MetadataRowReader.PrimaryKeys(result, "main", "default", "table").ToList(); + + Assert.Equal(new[] { 7, 1, 2 }, rows.Select(row => row.Sequence)); + Assert.Equal(new[] { "Main", "main", "main" }, rows.Select(row => row.Catalog)); + Assert.Equal(new[] { "Default", "default", "default" }, rows.Select(row => row.Schema)); + Assert.Equal(new[] { "Table", "table", "table" }, rows.Select(row => row.Table)); + Assert.Equal(new[] { "pk", "", "" }, rows.Select(row => row.Name)); + } + + [Fact] + public void ShowForeignKeys_FilterRawParentBeforeDefaults() + { + using var batch = Strings( + ("parentCatalogName", new string?[] { "Main", null }), + ("parentNamespace", new string?[] { "Sales", null }), + ("parentTableName", new string?[] { "Customers", null }), + ("col_name", new string?[] { "customer_id", "unknown" })); + var result = new MetadataBatches(new List { batch }, false); + + var row = Assert.Single(MetadataRowReader.ForeignKeys( + result, "main", "sales", "customers", "foreign", "default", "orders")); + + Assert.Equal("Main", row.ParentCatalog); + Assert.Equal("Sales", row.ParentSchema); + Assert.Equal("Customers", row.ParentTable); + Assert.Equal("foreign", row.Catalog); + Assert.Equal("default", row.Schema); + Assert.Equal("orders", row.Table); + Assert.Equal(1, row.Sequence); + Assert.Equal(0, row.UpdateRule); + Assert.Equal(0, row.DeleteRule); + Assert.Equal(5, row.Deferrability); + Assert.Equal("", row.Name); + Assert.Null(row.ParentName); + Assert.Equal(2, MetadataRowReader.ForeignKeys( + result, null, null, null, "foreign", "default", "orders").Count()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NativeKeys_DecodedAndFlatRowsPreserveServerFields(bool foreign) + { + var operation = foreign ? MetadataOperation.GetCrossReference : MetadataOperation.GetPrimaryKeys; + var schema = foreign ? MetadataSchemaFactory.CreateCrossReferenceSchema() : MetadataSchemaFactory.CreatePrimaryKeysSchema(); + schema = new Schema(schema.FieldsList.Reverse(), null); + var arrays = schema.FieldsList.Select(field => field.DataType.TypeId == ArrowTypeId.String + ? (IArrowArray)new StringArray.Builder().Append(field.Name switch + { + "TABLE_CAT" or "PKTABLE_CAT" => "Main", + "TABLE_SCHEM" or "PKTABLE_SCHEM" => "Sales", + "TABLE_NAME" or "PKTABLE_NAME" => "Customers", + _ => field.Name, + }).Build() + : new Int32Array.Builder().Append(7).Build()).ToArray(); + using var batch = new RecordBatch(schema, arrays, 1); + var response = new MetadataBatches(new List { batch }, true); + + var result = NativeMetadataResultBuilder.Build(response, schema, operation, + parentCatalog: "main", parentSchema: "sales", parentTable: "customers"); + using var stream = result.Stream!; + using var flat = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(7, ((Int32Array)flat.Column("KEQ_SEQ")).GetValue(0)); + Assert.Equal("PK_NAME", ((StringArray)flat.Column("PK_NAME")).GetString(0)); + + if (foreign) + { + var row = Assert.Single(MetadataRowReader.ForeignKeys( + response, "main", "sales", "customers", "foreign", "default", "orders")); + Assert.Equal("Main", row.ParentCatalog); + Assert.Equal("PK_NAME", row.ParentName); + Assert.Equal("FKTABLE_CAT", row.Catalog); + Assert.Equal(7, row.Sequence); + Assert.Empty(MetadataRowReader.ForeignKeys( + response, "other", "sales", "customers", "foreign", "default", "orders")); + var filtered = NativeMetadataResultBuilder.Build(response, schema, operation, parentCatalog: "other"); + using var filteredStream = filtered.Stream!; + Assert.Equal(0, filtered.RowCount); + } + else + { + var row = Assert.Single(MetadataRowReader.PrimaryKeys(response, "fallback", "fallback", "fallback")); + Assert.Equal("Main", row.Catalog); + Assert.Equal("Sales", row.Schema); + Assert.Equal("Customers", row.Table); + Assert.Equal("PK_NAME", row.Name); + Assert.Equal(7, row.Sequence); + } + } + private static RecordBatch ShowColumns(params (string Table, string Name)[] rows) => Strings( ("catalogName", rows.Select(_ => (string?)"main").ToArray()), diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs index cf8bb821..7fa8c3c6 100644 --- a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -43,9 +43,9 @@ public class NativeMetadataResultBuilderTests public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( string typeName, int? columnSize, int? decimalDigits, int expectedSize, int expectedScale) { - var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); + var target = new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Reverse(), null); Assert.Equal(24, target.FieldsList.Count); - var fields = target.FieldsList.Take(23).Select(field => + var fields = target.FieldsList.Where(field => field.Name != "BASE_TYPE_NAME").Select(field => field.Name == "ORDINAL_POSITION" ? new Field(field.Name, Int64Type.Default, true) : field).ToArray(); @@ -72,24 +72,24 @@ public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( using var nativeBatch = new RecordBatch(source, arrays.ToArray(), 1); var result = NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, target, MetadataOperation.GetColumns, - sourceCatalogs: new[] { "main" }); + new MetadataBatches(new List { nativeBatch }, true, "main"), + target, MetadataOperation.GetColumns); using var reader = result.Stream!; using var batch = await reader.ReadNextRecordBatchAsync(); Assert.NotNull(batch); - Assert.Equal(expectedSize, ((Int32Array)batch.Column(6)).GetValue(0)); - Assert.True(batch.Column(7).IsNull(0)); - Assert.Equal(expectedScale, ((Int32Array)batch.Column(8)).GetValue(0)); - Assert.Equal(0, ((Int32Array)batch.Column(16)).GetValue(0)); - Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); - Assert.Equal(ColumnMetadataHelper.GetBaseTypeName(typeName), ((StringArray)batch.Column(23)).GetString(0)); + Assert.Equal(expectedSize, ((Int32Array)batch.Column("COLUMN_SIZE")).GetValue(0)); + Assert.True(batch.Column("BUFFER_LENGTH").IsNull(0)); + Assert.Equal(expectedScale, ((Int32Array)batch.Column("DECIMAL_DIGITS")).GetValue(0)); + Assert.Equal(0, ((Int32Array)batch.Column("ORDINAL_POSITION")).GetValue(0)); + Assert.Equal("main", ((StringArray)batch.Column("TABLE_CAT")).GetString(0)); + Assert.Equal(ColumnMetadataHelper.GetBaseTypeName(typeName), ((StringArray)batch.Column("BASE_TYPE_NAME")).GetString(0)); } [Fact] public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCatalog() { - var schema = MetadataSchemaFactory.CreateTablesSchema(); + var schema = new Schema(MetadataSchemaFactory.CreateTablesSchema().FieldsList.Reverse(), null); var arrays = schema.FieldsList.Select(field => (IArrowArray)(field.Name switch { "TABLE_CAT" => new StringArray.Builder().AppendNull().Build(), @@ -99,14 +99,37 @@ public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCat using var nativeBatch = new RecordBatch(schema, arrays, 1); var result = NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, schema, MetadataOperation.GetTables, - requestedCatalog: "main", tableTypes: new[] { "TABLE" }); + new MetadataBatches(new List { nativeBatch }, true, "main", CatalogFilter.Exact("main")), + schema, MetadataOperation.GetTables, tableTypes: new[] { "TABLE" }); using var reader = result.Stream!; using var batch = await reader.ReadNextRecordBatchAsync(); Assert.Equal(1, result.RowCount); - Assert.Equal("main", ((StringArray)batch!.Column(0)).GetString(0)); - Assert.Equal("TABLE", ((StringArray)batch.Column(3)).GetString(0)); + Assert.Equal("main", ((StringArray)batch!.Column("TABLE_CAT")).GetString(0)); + Assert.Equal("TABLE", ((StringArray)batch.Column("TABLE_TYPE")).GetString(0)); + } + + [Fact] + public async Task GetSchemas_FillsMissingCatalogByFieldName() + { + var schema = new Schema(MetadataSchemaFactory.CreateSchemasSchema().FieldsList.Reverse(), null); + using var nativeBatch = new RecordBatch(schema, new IArrowArray[] + { + new StringArray.Builder().AppendNull().Build(), + new StringArray.Builder().Append("default").Build(), + }, 1); + var response = new MetadataBatches(new List { nativeBatch }, true, + "main", CatalogFilter.Exact("main")); + + var result = NativeMetadataResultBuilder.Build(response, schema, MetadataOperation.GetSchemas); + using var reader = result.Stream!; + using var batch = (await reader.ReadNextRecordBatchAsync())!; + + Assert.Equal("main", ((StringArray)batch.Column("TABLE_CATALOG")).GetString(0)); + Assert.Equal("default", ((StringArray)batch.Column("TABLE_SCHEM")).GetString(0)); + var decoded = Assert.Single(MetadataRowReader.Schemas(response)); + Assert.Equal("main", decoded.Catalog); + Assert.Equal("default", decoded.Schema); } [Theory] @@ -147,11 +170,11 @@ public async Task GetTables_FlatAndDecodedResultsShareCatalogFilteringAndOrderin }).ToArray(); using var nativeBatch = new RecordBatch(schema, arrays, tables.Length); string[]? tableTypes = filterTypes ? new[] { "TABLE" } : null; - var decoded = MetadataRowReader.Tables( - new MetadataBatches(new List { nativeBatch }, true), catalog, tableTypes); + var response = new MetadataBatches(new List { nativeBatch }, true, + catalog, CatalogFilter.Exact(catalog)); + var decoded = MetadataRowReader.Tables(response, tableTypes); var result = NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, schema, MetadataOperation.GetTables, - requestedCatalog: catalog, tableTypes: tableTypes); + response, schema, MetadataOperation.GetTables, tableTypes: tableTypes); using var reader = result.Stream!; using var batch = (await reader.ReadNextRecordBatchAsync())!; var names = (StringArray)batch.Column("TABLE_NAME"); @@ -199,8 +222,9 @@ public void NativeSchemasAndColumns_FilterLiteralCatalogsOnlyWhenRequested( using var nativeBatch = new RecordBatch(source, arrays, 2); var result = NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, target, operation, - requestedCatalog: requestedCatalog, requireExactCatalog: requireExactCatalog); + new MetadataBatches(new List { nativeBatch }, true, requestedCatalog, + CatalogFilter.Exact(requireExactCatalog ? requestedCatalog : null)), + target, operation); Assert.Equal(expectedRows, result.RowCount); } @@ -230,7 +254,8 @@ public async Task OtherNativeOperations_PreserveTheirSchemas(int operationCode) }).ToArray(); using var nativeBatch = new RecordBatch(schema, arrays, 1); - var result = NativeMetadataResultBuilder.Build(new[] { nativeBatch }, schema, operation); + var result = NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true), schema, operation); using var reader = result.Stream!; using var batch = await reader.ReadNextRecordBatchAsync(); @@ -260,7 +285,7 @@ public void GetColumns_RejectsWrongColumnNameOrType(bool wrongType) using var nativeBatch = new RecordBatch(schema, arrays, 1); Assert.Throws(() => NativeMetadataResultBuilder.Build( - new[] { nativeBatch }, target, MetadataOperation.GetColumns)); + new MetadataBatches(new List { nativeBatch }, true), target, MetadataOperation.GetColumns)); } } } diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 6d15e4ce..3a9a1d6e 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -15,6 +15,7 @@ */ using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Net; @@ -28,6 +29,7 @@ using Apache.Arrow.Types; using AdbcDrivers.Databricks.StatementExecution; using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; using AdbcDrivers.HiveServer2.Spark; using Microsoft.IO; using Moq; @@ -734,26 +736,22 @@ await Assert.ThrowsAsync( () => stmt.ExecuteQueryAsync(CancellationToken.None)); } - // ─── GetCrossReference parent-identifier filter ────────────────────────────── - // SHOW FOREIGN KEYS is scoped to the FOREIGN table only, so it returns FKs to - // every parent. GetCrossReferenceAsyncNoThrow filters the returned rows by any - // SPECIFIED (non-null) parent identifier — mirroring the JDBC reference driver - // (CrossReferenceKeysDatabricksResultSetAdapter.includeRow). ParentMatches is the - // per-field predicate; these pin the three behaviors the filter relies on: - // • null requested → no constraint (matches any row) — the GetColumnsExtended - // foreign-only reuse passes null parents and must stay UNFILTERED; - // • specified value → case-insensitive equality; - // • empty string → matches only an empty row value (no real FK has one), so - // an empty-string parent (the diff[6] case) filters every row out → empty. - // Reflection is used because the predicate is a private static helper (same - // approach as DatabricksStatementTests.GetConfOverlay). - private static bool InvokeParentMatches(string? requested, string? rowValue) + [Theory] + [InlineData((int)MetadataOperation.GetSchemas, "get_schemas")] + [InlineData((int)MetadataOperation.GetPrimaryKeys, "get_primary_keys")] + [InlineData((int)MetadataOperation.GetCrossReference, "get_cross_reference")] + public async Task MetadataRead_ObjectNotFound_ReturnsEmptyAndTraces(int operation, string name) { - var method = typeof(StatementExecutionStatement).GetMethod( - "ParentMatches", - System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); - Assert.NotNull(method); - return (bool)method!.Invoke(null, new object?[] { requested, rowValue })!; + using var activity = new Activity("metadata-test").Start(); + var exception = new DatabricksException("TABLE_OR_VIEW_NOT_FOUND"); + + int result = await StatementExecutionConnection.ReadMetadataOrEmptyAsync( + (MetadataOperation)operation, () => Task.FromException(exception), () => 0); + + Assert.Equal(0, result); + var traceEvent = Assert.Single(activity.Events); + Assert.Equal($"statement.{name}.object_not_found", traceEvent.Name); + Assert.Equal(exception.Message, Assert.Single(traceEvent.Tags).Value); } [Theory] @@ -761,27 +759,21 @@ private static bool InvokeParentMatches(string? requested, string? rowValue) [InlineData(null, "")] // null requested → matches even empty row value [InlineData("comparator_tests", "comparator_tests")] // exact match [InlineData("COMPARATOR_TESTS", "comparator_tests")] // case-insensitive match - public void ParentMatches_NullOrEqual_ReturnsTrue(string? requested, string rowValue) + public void ExactCatalogFilter_NullOrEqual_ReturnsTrue(string? requested, string rowValue) { - Assert.True(InvokeParentMatches(requested, rowValue)); + Assert.True(MetadataRowReader.MatchesCatalog(CatalogFilter.Exact(requested), rowValue)); } [Theory] [InlineData("", "comparator_tests")] // empty-string requested (diff[6]) → filters out [InlineData("other_catalog", "comparator_tests")] // specified, non-matching → filters out [InlineData("comparator_tests", null)] // specified vs NULL server column → filters out - public void ParentMatches_SpecifiedNonMatching_ReturnsFalse(string requested, string? rowValue) + public void ExactCatalogFilter_SpecifiedNonMatching_ReturnsFalse(string requested, string? rowValue) { - Assert.False(InvokeParentMatches(requested, rowValue)); + Assert.False(MetadataRowReader.MatchesCatalog(CatalogFilter.Exact(requested), rowValue)); } - // ─── GetCrossReference parent-filter WIRING (end-to-end at the Http seam) ───── - // The tests above pin the ParentMatches predicate in isolation. These exercise the - // full SEA row-filtering loop in GetCrossReferenceAsyncNoThrow: a mocked SHOW FOREIGN - // KEYS result carrying FKs to TWO different parents flows through server-value - // extraction → ParentMatches × 3 → continue → value population. A regression that - // filtered on the wrong variable, or that let the empty-parent case (diff[6]) leak - // rows instead of collapsing to zero, would pass the predicate suite but fail here. + // Exercise parent filtering through the HTTP metadata path, not just the predicate. // A SHOW FOREIGN KEYS result with two rows: orders(customer_id) → main.sales.customers // and orders(manager_id) → main.hr.employees. Both share the same FOREIGN table diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs index 3754679a..2d30d7ab 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs @@ -21,6 +21,7 @@ using System.Net; using System.Net.Http; using System.Text.Json; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using AdbcDrivers.Databricks.StatementExecution; @@ -60,10 +61,9 @@ public class StatementExecutionMetadataResponseTests }; foreach (int mode in new[] { 0, 1 }) { - foreach (var (pattern, matches) in patterns) - yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.All, pattern, matches }; - foreach (var (pattern, matches) in patterns.Where(entry => entry.Pattern == null || entry.Pattern == "main")) - yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.Tables, pattern, matches }; + foreach (var depth in new[] { AdbcConnection.GetObjectsDepth.All, AdbcConnection.GetObjectsDepth.Tables }) + foreach (var (pattern, matches) in patterns) + yield return new object?[] { mode, depth, pattern, matches }; } foreach (int mode in new[] { 2, 3 }) yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.All, null, s_catalogs }; @@ -71,7 +71,7 @@ public class StatementExecutionMetadataResponseTests [Theory] [MemberData(nameof(CatalogPatterns))] - public async Task GetObjects_PreservesExistingCatalogScope( + public async Task GetObjects_NativeCatalogPatternsPreserveChildrenAndShowScopes( int mode, AdbcConnection.GetObjectsDepth depth, string? pattern, string[] expectedCatalogs) { List statements = new List(); @@ -86,7 +86,8 @@ public async Task GetObjects_PreservesExistingCatalogScope( catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); foreach (AdbcCatalog catalog in catalogs) { - if (pattern != null && !string.Equals(pattern, catalog.Name, StringComparison.OrdinalIgnoreCase)) + if (mode == 0 && pattern != null && + !string.Equals(pattern, catalog.Name, StringComparison.OrdinalIgnoreCase)) { Assert.Empty(catalog.DbSchemas!); continue; @@ -107,6 +108,12 @@ public async Task GetObjects_PreservesExistingCatalogScope( int catalogQueries = pattern == null && depth == AdbcConnection.GetObjectsDepth.All ? 2 : 1; Assert.Equal(catalogQueries, statements.Count(sql => sql.StartsWith("SHOW CATALOGS", StringComparison.Ordinal))); + Assert.Single(statements, sql => sql.StartsWith("SHOW SCHEMAS", StringComparison.Ordinal)); + Assert.Single(statements, sql => sql.StartsWith("SHOW TABLES", StringComparison.Ordinal)); + int columnQueries = depth != AdbcConnection.GetObjectsDepth.All ? 0 + : pattern == null ? s_catalogs.Length : 1; + Assert.Equal(columnQueries, + statements.Count(sql => sql.StartsWith("SHOW COLUMNS", StringComparison.Ordinal))); if (pattern != null) { string scope = $"`{pattern.ToLowerInvariant()}`"; @@ -295,8 +302,10 @@ private static HttpClient CreateHttpClient( string operation = request.Headers.GetValues("x-databricks-metadata-operation-type").Single(); bool allCatalogs = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs"; string? requestedCatalog = allCatalogs - ? null : s_catalogs.SingleOrDefault(catalog => sql.Contains($"`{catalog}`")); - if (!allCatalogs && requestedCatalog == null) + ? null : Regex.Match(sql, @" IN (?:CATALOG )?`((?:``|[^`])*)`").Groups[1].Value.Replace("``", "`"); + bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || + (mode == 3 && (operation != "GetColumns" || requestedCatalog == "main")); + if (!native && !allCatalogs && !s_catalogs.Contains(requestedCatalog)) { return Response(JsonSerializer.Serialize(new { @@ -309,13 +318,9 @@ private static HttpClient CreateHttpClient( }, })); } - bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || - (mode == 3 && (operation != "GetColumns" || requestedCatalog == "main")); - string[] catalogs = requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; + string[] catalogs = native || requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; if (!native && operation == "GetCatalogs") catalogs = matchingCatalogs ?? s_catalogs; - if (native && requestedCatalog != null && operation is "GetTables" or "GetColumns") - catalogs = catalogs.Concat(new[] { requestedCatalog == "other" ? "main" : "other" }).ToArray(); using RecordBatch batch = CreateBatch(operation, native, catalogs, requestedCatalog); using MemoryStream raw = new MemoryStream(); @@ -393,8 +398,7 @@ private static RecordBatch CreateBatch( { string? value = field.Name switch { - "TABLE_CAT" or "catalog" or "catalogName" => catalog, - "TABLE_CATALOG" => requestedCatalog == null ? catalog : null, + "TABLE_CAT" or "TABLE_CATALOG" or "catalog" or "catalogName" => catalog, "TABLE_SCHEM" or "databaseName" or "namespace" => "default", "TABLE_NAME" or "tableName" => "t1", "TABLE_TYPE" or "tableType" => "TABLE", diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs index 2297c8eb..54aac053 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -185,8 +185,8 @@ public async Task ColumnFanout_KeepsTheQueriedCatalog() var result = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); - Assert.Equal(2, result.Batches.Count); - Assert.Equal(new[] { "main", "other" }, result.SourceCatalogs); + Assert.Equal(2, result.Results.Count); + Assert.Equal(new[] { "main", "other" }, result.Results.Select(response => response.SourceCatalog)); } [Theory] @@ -249,14 +249,13 @@ public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() var columns = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); Assert.True(columns.IsNative); - Assert.Equal(2, columns.Batches.Count); - Assert.All(columns.Batches, batch => Assert.Equal(2, batch.Length)); + Assert.Equal(2, columns.Results.Count); + Assert.All(columns.Results, response => Assert.Equal(2, Assert.Single(response.Batches).Length)); Assert.Equal(new[] { "foo_bar", "fooxbar" }, columns.Rows.Select(row => row.Catalog)); var result = NativeMetadataResultBuilder.Build( - columns.Batches, - MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns, - sourceCatalogs: columns.SourceCatalogs, requireExactSourceCatalog: true); + columns.Results, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); using var stream = result.Stream!; Assert.Equal(2, result.RowCount); From f0ccb1d0ee79d53cf0d3499507209848bd08c0c2 Mon Sep 17 00:00:00 2001 From: Vu Anh Phung Date: Fri, 9 Oct 2026 14:50:30 +0000 Subject: [PATCH 22/22] fix(csharp): preserve native metadata patterns and mixed column fields Signed-off-by: Vu Anh Phung --- csharp/src/FlatColumnsResultBuilder.cs | 58 +++++++++++- .../StatementExecution/MetadataRowReader.cs | 12 ++- .../StatementExecutionStatement.cs | 31 ++----- .../MetadataRowReaderTests.cs | 9 +- ...StatementExecutionMetadataResponseTests.cs | 92 ++++++++++++++++--- 5 files changed, 164 insertions(+), 38 deletions(-) diff --git a/csharp/src/FlatColumnsResultBuilder.cs b/csharp/src/FlatColumnsResultBuilder.cs index 064593bb..994ce482 100644 --- a/csharp/src/FlatColumnsResultBuilder.cs +++ b/csharp/src/FlatColumnsResultBuilder.cs @@ -15,7 +15,11 @@ */ using System.Collections.Generic; -using AdbcDrivers.Databricks.StatementExecution;using AdbcDrivers.HiveServer2; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; using AdbcDrivers.HiveServer2.Hive2; using Apache.Arrow; using Apache.Arrow.Adbc; @@ -29,6 +33,58 @@ namespace AdbcDrivers.Databricks /// internal static class FlatColumnsResultBuilder { + internal static async Task BuildFlatColumnsResultAsync( + ColumnMetadataResult columns, CancellationToken cancellationToken) + { + var schema = MetadataSchemaFactory.CreateColumnMetadataSchema(); + if (columns.IsNative) + return NativeMetadataResultBuilder.Build(columns.Results, schema, MetadataOperation.GetColumns); + if (!columns.Results.Any(result => result.IsNative && result.Batches.Count > 0)) + return BuildFlatColumnsResult(columns.Rows); + + // Normalize each response independently so SHOW fallback cannot discard native fields. + var batches = new List(); + try + { + for (int index = 0; index < columns.Results.Count; index++) + { + var result = columns.Results[index]; + var flat = result.IsNative + ? NativeMetadataResultBuilder.Build(result, schema, MetadataOperation.GetColumns) + : BuildFlatColumnsResult(columns.RowsByResult[index]); + using var stream = flat.Stream!; + var batch = await stream.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); + if (batch != null) batches.Add(batch); + } + + var arrays = Enumerable.Range(0, schema.FieldsList.Count) + .Select(column => ArrowArrayFactory.BuildArray(ArrayDataConcatenator.Concatenate( + batches.Select(batch => batch.Column(column).Data).ToList())!)) + .ToArray(); + return new QueryResult(batches.Sum(batch => batch.Length), new HiveInfoArrowStream(schema, arrays)); + } + finally + { + foreach (var batch in batches) batch.Dispose(); + } + } + + private static QueryResult BuildFlatColumnsResult(IEnumerable columns) + { + var tableInfos = new Dictionary(); + foreach (var column in columns) + { + if (column.Catalog == null || column.Schema == null || column.Table == null) continue; + string key = $"{column.Catalog}.{column.Schema}.{column.Table}"; + if (!tableInfos.ContainsKey(key)) + tableInfos[key] = (column.Catalog, column.Schema, column.Table, new TableInfo("TABLE")); + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfos[key].info, column.Name, column.TypeName, column.Ordinal, column.Nullable, + columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); + } + return BuildFlatColumnsResult(tableInfos.Values); + } + internal static QueryResult BuildFlatColumnsResult( IEnumerable<(string catalog, string schema, string table, TableInfo info)> tables) { diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs index bd8c2438..08727504 100644 --- a/csharp/src/StatementExecution/MetadataRowReader.cs +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -64,11 +64,13 @@ internal ColumnMetadataResult(IReadOnlyList results, bool requi Results = results; IsNative = results.Any(result => result.Batches.Count > 0) && results.Where(result => result.Batches.Count > 0).All(result => result.IsNative); - Rows = MetadataRowReader.Columns(results, requireTableIdentifiers).ToList(); + RowsByResult = MetadataRowReader.ColumnGroups(results, requireTableIdentifiers).ToList(); + Rows = RowsByResult.SelectMany(rows => rows).ToList(); } internal IReadOnlyList Results { get; } internal bool IsNative { get; } + internal IReadOnlyList> RowsByResult { get; } internal IReadOnlyList Rows { get; } } @@ -346,10 +348,15 @@ internal static List Tables( internal static IEnumerable Columns(IEnumerable results, bool requireTableIdentifiers = false) + => ColumnGroups(results, requireTableIdentifiers).SelectMany(rows => rows); + + internal static IEnumerable> ColumnGroups(IEnumerable results, + bool requireTableIdentifiers = false) { var positions = new Dictionary(); foreach (var result in results) { + var group = new List(); var rows = result.IsNative ? NativeColumns(result) : result.Batches.SelectMany(batch => ShowColumns(batch, positions)); foreach (var row in rows) @@ -360,8 +367,9 @@ internal static IEnumerable Columns(IEnumerable resu string key = $"{row.Catalog}.{row.Schema}.{row.Table}"; positions.TryGetValue(key, out int position); positions[key] = position + 1; - yield return row; + group.Add(row); } + yield return group; } } diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 4cea7bcb..5b070f9b 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -1389,7 +1389,7 @@ private async Task GetTablesAsync(CancellationToken cancellationTok var result = await _connection.ReadMetadataAsync( sql, MetadataOperation.GetTables, cancellationToken, catalog, - CatalogFilter.Exact(catalog)).ConfigureAwait(false); + _escapePatternWildcards ? CatalogFilter.Exact(catalog) : CatalogFilter.Pattern(catalog)).ConfigureAwait(false); IsNativeMetadataResult = result.IsNative; // Issue #526: match JDBC's MetadataResultSetBuilder and the Thrift path - @@ -1477,28 +1477,13 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo () => new ColumnMetadataResult(System.Array.Empty())).ConfigureAwait(false); IsNativeMetadataResult = columns.IsNative; - if (IsNativeMetadataResult) - { - return NativeMetadataResultBuilder.Build( - columns.Results, - MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); - } - - var tableInfos = new Dictionary(); - - foreach (var column in columns.Rows) - { - if (column.Catalog == null || column.Schema == null || column.Table == null) continue; - string key = $"{column.Catalog}.{column.Schema}.{column.Table}"; - if (!tableInfos.ContainsKey(key)) - tableInfos[key] = (column.Catalog, column.Schema, column.Table, new TableInfo("TABLE")); - ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfos[key].info, column.Name, column.TypeName, column.Ordinal, column.Nullable, - columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); - } - - activity?.SetTag("result_tables", tableInfos.Count); - return FlatColumnsResultBuilder.BuildFlatColumnsResult(tableInfos.Values); + if (!columns.IsNative) + activity?.SetTag("result_tables", columns.Rows + .Where(column => column.Catalog != null && column.Schema != null && column.Table != null) + .Select(column => $"{column.Catalog}.{column.Schema}.{column.Table}") + .Distinct().Count()); + return await FlatColumnsResultBuilder.BuildFlatColumnsResultAsync( + columns, cancellationToken).ConfigureAwait(false); }, "GetColumns").ConfigureAwait(false); } diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs index 886fec05..63e751ae 100644 --- a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -196,7 +196,7 @@ public async Task MissingNativeCatalog_OnlyUsesConcreteScope( } [Fact] - public void MixedColumns_PreserveServerOrdinalsAndCountShowRows() + public async Task MixedColumns_PreserveServerOrdinalsAndCountShowRows() { using var native = NativeColumns("main"); using var show = ShowColumns(("t", "b")); @@ -209,6 +209,13 @@ public void MixedColumns_PreserveServerOrdinalsAndCountShowRows() Assert.False(result.IsNative); Assert.Equal(new[] { 7, 1 }, result.Rows.Select(row => row.Ordinal)); Assert.Equal(new[] { "a", "b" }, result.Rows.Select(row => row.Name)); + + var flat = await FlatColumnsResultBuilder.BuildFlatColumnsResultAsync( + result, System.Threading.CancellationToken.None); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + var positions = (Int32Array)output.Column("ORDINAL_POSITION"); + Assert.Equal(new[] { 7, 1 }, Enumerable.Range(0, output.Length).Select(row => positions.GetValue(row)!.Value)); } [Fact] diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs index 2d30d7ab..0bd3f4e0 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs @@ -169,10 +169,43 @@ public void MetadataCommand_ResetsNativeFlagForSyntheticResults() Assert.False(statement.IsNativeMetadataResult); } - [Fact] - public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdinals() + [Theory] + [InlineData("main", false, "main")] + [InlineData("ma%", false, "main,marketing")] + [InlineData("MA%", false, "main,marketing")] + [InlineData("ma_n", false, "main")] + [InlineData("foo_bar", false, "foo_bar,fooxbar")] + [InlineData(@"foo\_bar", false, "foo_bar")] + [InlineData("ma%", true, "")] + [InlineData("foo_bar", true, "foo_bar")] + [InlineData("%", true, "")] + public async Task GetTables_NativeCatalogPatternsRespectLiteralMode( + string catalog, bool literal, string expectedCatalogs) { - using HttpClient http = CreateHttpClient(3, new List()); + var statements = new List(); + using HttpClient http = CreateHttpClient(1, statements); + using StatementExecutionConnection connection = CreateConnection(http); + using var statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, catalog); + statement.SetOption(ApacheParameters.EscapePatternWildcards, literal.ToString()); + statement.SqlQuery = "GetTables"; + using var stream = statement.ExecuteQuery().Stream!; + using var batch = (await stream.ReadNextRecordBatchAsync())!; + + Assert.True(statement.IsNativeMetadataResult); + Assert.Equal(expectedCatalogs.Length == 0 ? System.Array.Empty() : expectedCatalogs.Split(','), + Enumerable.Range(0, batch.Length).Select(row => ((StringArray)batch.Column("TABLE_CAT")).GetString(row))); + Assert.Contains($"IN CATALOG `{catalog}`", Assert.Single(statements)); + } + + [Theory] + [InlineData("main")] + [InlineData("marketing")] + public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdinals(string nativeCatalog) + { + using HttpClient http = CreateHttpClient(3, new List(), + nativeColumnsCatalog: nativeCatalog, columnType: "STRUCT"); using StatementExecutionConnection connection = CreateConnection(http); using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); @@ -186,12 +219,45 @@ public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdin var ordinals = (Int32Array)batch.Column("ORDINAL_POSITION"); var defaults = (StringArray)batch.Column("COLUMN_DEF"); var autoIncrement = (StringArray)batch.Column("IS_AUTO_INCREMENT"); + Assert.Equal(s_catalogs, Enumerable.Range(0, batch.Length).Select(row => catalogs.GetString(row))); + + using HttpClient nativeHttp = CreateHttpClient(1, new List(), columnType: "STRUCT"); + using var nativeConnection = CreateConnection(nativeHttp); + using var nativeStatement = nativeConnection.CreateStatement(); + nativeStatement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + nativeStatement.SetOption(ApacheParameters.CatalogName, nativeCatalog); + nativeStatement.SetOption(ApacheParameters.EscapePatternWildcards, "true"); + nativeStatement.SqlQuery = "GetColumns"; + using var nativeStream = nativeStatement.ExecuteQuery().Stream!; + using var nativeBatch = (await nativeStream.ReadNextRecordBatchAsync())!; + Assert.Equal(1, nativeBatch.Length); + for (int row = 0; row < batch.Length; row++) { Assert.Equal(0, ordinals.GetValue(row)); - bool native = catalogs.GetString(row) == "main"; + bool native = catalogs.GetString(row) == nativeCatalog; Assert.Equal(native ? "7" : null, defaults.GetString(row)); Assert.Equal(native ? "YES" : "NO", autoIncrement.GetString(row)); + Assert.Equal(native ? 4 : 0, ((Int32Array)batch.Column("COLUMN_SIZE")).GetValue(row)); + Assert.Equal(native ? "server comment" : "", ((StringArray)batch.Column("REMARKS")).GetString(row)); + if (!native) continue; + for (int column = 0; column < batch.ColumnCount; column++) + { + switch (batch.Column(column)) + { + case StringArray strings: + Assert.Equal(((StringArray)nativeBatch.Column(column)).GetString(0), strings.GetString(row)); + break; + case Int16Array numbers: + Assert.Equal(((Int16Array)nativeBatch.Column(column)).GetValue(0), numbers.GetValue(row)); + break; + case Int32Array numbers: + Assert.Equal(((Int32Array)nativeBatch.Column(column)).GetValue(0), numbers.GetValue(row)); + break; + default: + throw new InvalidOperationException($"Unexpected metadata type: {batch.Column(column).Data.DataType}"); + } + } } } @@ -280,7 +346,8 @@ private static StatementExecutionConnection CreateConnection(HttpClient http, bo } private static HttpClient CreateHttpClient( - int mode, List statements, string[]? matchingCatalogs = null, bool prependEmptyShowColumn = false) + int mode, List statements, string[]? matchingCatalogs = null, bool prependEmptyShowColumn = false, + string nativeColumnsCatalog = "main", string columnType = "INT") { Mock handler = new Mock(); handler.Protected() @@ -304,7 +371,7 @@ private static HttpClient CreateHttpClient( string? requestedCatalog = allCatalogs ? null : Regex.Match(sql, @" IN (?:CATALOG )?`((?:``|[^`])*)`").Groups[1].Value.Replace("``", "`"); bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || - (mode == 3 && (operation != "GetColumns" || requestedCatalog == "main")); + (mode == 3 && (operation != "GetColumns" || requestedCatalog == nativeColumnsCatalog)); if (!native && !allCatalogs && !s_catalogs.Contains(requestedCatalog)) { return Response(JsonSerializer.Serialize(new @@ -322,13 +389,14 @@ private static HttpClient CreateHttpClient( if (!native && operation == "GetCatalogs") catalogs = matchingCatalogs ?? s_catalogs; - using RecordBatch batch = CreateBatch(operation, native, catalogs, requestedCatalog); + using RecordBatch batch = CreateBatch(operation, native, catalogs, requestedCatalog, columnType: columnType); using MemoryStream raw = new MemoryStream(); using (ArrowStreamWriter writer = new ArrowStreamWriter(raw, batch.Schema)) { if (!native && operation == "GetColumns" && prependEmptyShowColumn) { - using var empty = CreateBatch(operation, false, catalogs, requestedCatalog, columnName: ""); + using var empty = CreateBatch(operation, false, catalogs, requestedCatalog, + columnName: "", columnType: columnType); writer.WriteRecordBatch(empty); } writer.WriteRecordBatch(batch); @@ -364,7 +432,8 @@ private static HttpResponseMessage Response(string body) => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; private static RecordBatch CreateBatch( - string operation, bool native, string[] catalogs, string? requestedCatalog, string columnName = "a") + string operation, bool native, string[] catalogs, string? requestedCatalog, + string columnName = "a", string columnType = "INT") { Schema schema; if (native) @@ -403,8 +472,9 @@ private static RecordBatch CreateBatch( "TABLE_NAME" or "tableName" => "t1", "TABLE_TYPE" or "tableType" => "TABLE", "COLUMN_NAME" or "col_name" => columnName, - "TYPE_NAME" or "columnType" => "INT", + "TYPE_NAME" or "columnType" => columnType, "COLUMN_DEF" => "7", + "REMARKS" => "server comment", "IS_AUTO_INCREMENT" => "YES", "isNullable" => "true", _ => null, @@ -413,7 +483,7 @@ private static RecordBatch CreateBatch( } return (IArrowArray)strings.Build(); } - int number = field.Name == "NULLABLE" ? 1 : field.Name == "DATA_TYPE" ? 4 : 0; + int number = field.Name == "NULLABLE" ? 1 : field.Name is "DATA_TYPE" or "COLUMN_SIZE" ? 4 : 0; return field.DataType.TypeId switch { ArrowTypeId.Int16 => (IArrowArray)new Int16Array.Builder().AppendRange(