diff --git a/csharp/src/FlatColumnsResultBuilder.cs b/csharp/src/FlatColumnsResultBuilder.cs index 30ae73e0..994ce482 100644 --- a/csharp/src/FlatColumnsResultBuilder.cs +++ b/csharp/src/FlatColumnsResultBuilder.cs @@ -15,7 +15,11 @@ */ using System.Collections.Generic; -using AdbcDrivers.Databricks.StatementExecution;using AdbcDrivers.HiveServer2; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; using AdbcDrivers.HiveServer2.Hive2; using Apache.Arrow; using Apache.Arrow.Adbc; @@ -29,6 +33,58 @@ namespace AdbcDrivers.Databricks /// internal static class FlatColumnsResultBuilder { + internal static async Task BuildFlatColumnsResultAsync( + ColumnMetadataResult columns, CancellationToken cancellationToken) + { + var schema = MetadataSchemaFactory.CreateColumnMetadataSchema(); + if (columns.IsNative) + return NativeMetadataResultBuilder.Build(columns.Results, schema, MetadataOperation.GetColumns); + if (!columns.Results.Any(result => result.IsNative && result.Batches.Count > 0)) + return BuildFlatColumnsResult(columns.Rows); + + // Normalize each response independently so SHOW fallback cannot discard native fields. + var batches = new List(); + try + { + for (int index = 0; index < columns.Results.Count; index++) + { + var result = columns.Results[index]; + var flat = result.IsNative + ? NativeMetadataResultBuilder.Build(result, schema, MetadataOperation.GetColumns) + : BuildFlatColumnsResult(columns.RowsByResult[index]); + using var stream = flat.Stream!; + var batch = await stream.ReadNextRecordBatchAsync(cancellationToken).ConfigureAwait(false); + if (batch != null) batches.Add(batch); + } + + var arrays = Enumerable.Range(0, schema.FieldsList.Count) + .Select(column => ArrowArrayFactory.BuildArray(ArrayDataConcatenator.Concatenate( + batches.Select(batch => batch.Column(column).Data).ToList())!)) + .ToArray(); + return new QueryResult(batches.Sum(batch => batch.Length), new HiveInfoArrowStream(schema, arrays)); + } + finally + { + foreach (var batch in batches) batch.Dispose(); + } + } + + private static QueryResult BuildFlatColumnsResult(IEnumerable columns) + { + var tableInfos = new Dictionary(); + foreach (var column in columns) + { + if (column.Catalog == null || column.Schema == null || column.Table == null) continue; + string key = $"{column.Catalog}.{column.Schema}.{column.Table}"; + if (!tableInfos.ContainsKey(key)) + tableInfos[key] = (column.Catalog, column.Schema, column.Table, new TableInfo("TABLE")); + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfos[key].info, column.Name, column.TypeName, column.Ordinal, column.Nullable, + columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); + } + return BuildFlatColumnsResult(tableInfos.Values); + } + internal static QueryResult BuildFlatColumnsResult( IEnumerable<(string catalog, string schema, string table, TableInfo info)> tables) { @@ -81,7 +137,8 @@ internal static QueryResult BuildFlatColumnsResult( nullableBuilder.Append(info.Nullable[i]); remarksBuilder.Append(""); - columnDefBuilder.AppendNull(); + if (string.IsNullOrEmpty(info.ColumnDefault[i])) columnDefBuilder.AppendNull(); + else columnDefBuilder.Append(info.ColumnDefault[i]); sqlDataTypeBuilder.Append(info.ColType[i]); diff --git a/csharp/src/StatementExecution/ColumnMetadataHelper.cs b/csharp/src/StatementExecution/ColumnMetadataHelper.cs index b3755cb7..cfe103f9 100644 --- a/csharp/src/StatementExecution/ColumnMetadataHelper.cs +++ b/csharp/src/StatementExecution/ColumnMetadataHelper.cs @@ -184,6 +184,24 @@ internal static string GetBaseTypeName(string typeName) }; } + internal static (long ColumnSize, long DecimalDigits) NormalizePrecisionAndScale( + string? typeName, long? columnSize, long? decimalDigits) + { + if (typeName != null) + { + string baseName = GetBaseTypeName(typeName); + if (baseName is "DECIMAL" or "NUMERIC") + { + columnSize = GetColumnSizeDefault(typeName); + decimalDigits = GetDecimalDigitsDefault(typeName); + } + else if (s_charBaseTypes.Contains(baseName)) + columnSize = GetColumnSizeDefault(typeName); + } + // Thrift defaults null sizes and digits to zero before type-specific overrides. + return (columnSize ?? 0, decimalDigits ?? 0); + } + internal static int? GetBufferLength(string typeName) { string baseName = GetBaseTypeName(typeName); @@ -242,19 +260,21 @@ internal static void PopulateTableInfoFromTypeName( int ordinalPosition, bool isNullable = true, string? comment = null, - string? columnDefault = null) + string? columnDefault = null, + bool isAutoIncrement = false) { tableInfo.ColumnName.Add(columnName); tableInfo.TypeName.Add(typeName); tableInfo.ColType.Add(GetDataTypeCode(typeName)); tableInfo.BaseTypeName.Add(GetBaseTypeName(typeName)); - tableInfo.Precision.Add(GetColumnSizeDefault(typeName)); - int? scale = GetDecimalDigitsDefault(typeName); - tableInfo.Scale.Add(scale.HasValue ? (short)scale.Value : null); + var precisionAndScale = NormalizePrecisionAndScale( + typeName, GetColumnSizeDefault(typeName), GetDecimalDigitsDefault(typeName)); + tableInfo.Precision.Add(checked((int)precisionAndScale.ColumnSize)); + tableInfo.Scale.Add((short)precisionAndScale.DecimalDigits); tableInfo.OrdinalPosition.Add(ordinalPosition); tableInfo.Nullable.Add(isNullable ? (short)1 : (short)0); tableInfo.IsNullable.Add(isNullable ? "YES" : "NO"); - tableInfo.IsAutoIncrement.Add(false); + tableInfo.IsAutoIncrement.Add(isAutoIncrement); tableInfo.ColumnDefault.Add(columnDefault ?? ""); } diff --git a/csharp/src/StatementExecution/MetadataRowReader.cs b/csharp/src/StatementExecution/MetadataRowReader.cs new file mode 100644 index 00000000..08727504 --- /dev/null +++ b/csharp/src/StatementExecution/MetadataRowReader.cs @@ -0,0 +1,599 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal readonly struct CatalogFilter + { + private CatalogFilter(string? value, bool isPattern) + { + Value = value; + IsPattern = isPattern; + } + + internal string? Value { get; } + internal bool IsPattern { get; } + + internal static CatalogFilter Exact(string? catalog) => new(catalog, false); + internal static CatalogFilter Pattern(string? pattern) => new(pattern, true); + } + + internal sealed class MetadataBatches + { + internal MetadataBatches(List batches, bool isNative, + string? sourceCatalog = null, CatalogFilter catalogFilter = default) + { + Batches = batches; + IsNative = isNative; + SourceCatalog = sourceCatalog; + CatalogFilter = catalogFilter; + } + + internal List Batches { get; } + internal bool IsNative { get; } + internal string? SourceCatalog { get; } + internal CatalogFilter CatalogFilter { get; } + } + + internal sealed class ColumnMetadataResult + { + internal ColumnMetadataResult(IReadOnlyList results, bool requireTableIdentifiers = false) + { + Results = results; + IsNative = results.Any(result => result.Batches.Count > 0) && + results.Where(result => result.Batches.Count > 0).All(result => result.IsNative); + RowsByResult = MetadataRowReader.ColumnGroups(results, requireTableIdentifiers).ToList(); + Rows = RowsByResult.SelectMany(rows => rows).ToList(); + } + + internal IReadOnlyList Results { get; } + internal bool IsNative { get; } + internal IReadOnlyList> RowsByResult { get; } + internal IReadOnlyList Rows { get; } + } + + internal readonly struct SchemaRow + { + internal SchemaRow(string catalog, string schema) + { + Catalog = catalog; + Schema = schema; + } + + internal string Catalog { get; } + internal string Schema { get; } + } + + internal readonly struct TableRow + { + internal TableRow(string catalog, string schema, string table, string tableType, string remarks) + { + Catalog = catalog; + Schema = schema; + Table = table; + TableType = tableType; + Remarks = remarks; + } + + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string TableType { get; } + internal string Remarks { get; } + } + + internal readonly struct ColumnRow + { + internal ColumnRow(string? catalog, string? schema, string? table, string name, + string typeName, bool nullable, int ordinal, string? columnDefault, bool isAutoIncrement) + { + Catalog = catalog; + Schema = schema; + Table = table; + Name = name; + TypeName = typeName; + Nullable = nullable; + Ordinal = ordinal; + Default = columnDefault; + IsAutoIncrement = isAutoIncrement; + } + + internal string? Catalog { get; } + internal string? Schema { get; } + internal string? Table { get; } + internal string Name { get; } + internal string TypeName { get; } + internal bool Nullable { get; } + internal int Ordinal { get; } + internal string? Default { get; } + internal bool IsAutoIncrement { get; } + } + + internal readonly struct PrimaryKeyRow + { + internal PrimaryKeyRow(string catalog, string schema, string table, string column, int sequence, string name) + { + Catalog = catalog; + Schema = schema; + Table = table; + Column = column; + Sequence = sequence; + Name = name; + } + + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string Column { get; } + internal int Sequence { get; } + internal string Name { get; } + } + + internal readonly struct ForeignKeyRow + { + internal ForeignKeyRow(string parentCatalog, string parentSchema, string parentTable, string parentColumn, + string catalog, string schema, string table, string column, int sequence, + int updateRule, int deleteRule, string name, string? parentName, int deferrability) + { + ParentCatalog = parentCatalog; + ParentSchema = parentSchema; + ParentTable = parentTable; + ParentColumn = parentColumn; + Catalog = catalog; + Schema = schema; + Table = table; + Column = column; + Sequence = sequence; + UpdateRule = updateRule; + DeleteRule = deleteRule; + Name = name; + ParentName = parentName; + Deferrability = deferrability; + } + + internal string ParentCatalog { get; } + internal string ParentSchema { get; } + internal string ParentTable { get; } + internal string ParentColumn { get; } + internal string Catalog { get; } + internal string Schema { get; } + internal string Table { get; } + internal string Column { get; } + internal int Sequence { get; } + internal int UpdateRule { get; } + internal int DeleteRule { get; } + internal string Name { get; } + internal string? ParentName { get; } + internal int Deferrability { get; } + } + + internal readonly struct NativeMetadataRow + { + internal NativeMetadataRow(NativeMetadataColumns columns, int index, string? sourceCatalog) + { + Columns = columns; + Index = index; + SourceCatalog = sourceCatalog; + } + + private NativeMetadataColumns Columns { get; } + private int Index { get; } + internal string? SourceCatalog { get; } + internal string? String(string name) => Columns.String(name, Index); + internal long? Integer(string name) => Columns.Integer(name, Index); + } + + internal static class MetadataRowReader + { + internal static List NativeRows( + IEnumerable results, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + { + string? catalogField = operation switch + { + MetadataOperation.GetCatalogs => "TABLE_CAT", + MetadataOperation.GetSchemas => "TABLE_CATALOG", + MetadataOperation.GetTables or MetadataOperation.GetColumns => "TABLE_CAT", + _ => null, + }; + var rows = new List(); + foreach (var result in results) + { + // A wildcard scope cannot supply a missing native catalog identifier. + string? sourceCatalog = result.CatalogFilter.IsPattern + ? LiteralCatalog(result.SourceCatalog) : result.SourceCatalog; + foreach (var batch in result.Batches) + { + var columns = new NativeMetadataColumns(batch, schema, operation); + for (int index = 0; index < batch.Length; index++) + { + var row = new NativeMetadataRow(columns, index, sourceCatalog); + if (catalogField != null && + !MatchesCatalog(result.CatalogFilter, row.String(catalogField) ?? row.SourceCatalog)) + continue; + if (operation == MetadataOperation.GetTables && tableTypes != null && + !tableTypes.Contains(DefaultTableType(row.String("TABLE_TYPE")))) + continue; + if (operation == MetadataOperation.GetCrossReference && + !MatchesParent(parentCatalog, parentSchema, parentTable, + row.String("PKTABLE_CAT"), row.String("PKTABLE_SCHEM"), row.String("PKTABLE_NAME"))) + continue; + rows.Add(row); + } + } + } + + // Thrift orders tables by type, catalog, schema, then name. + if (operation == MetadataOperation.GetTables) + rows.Sort((left, right) => CompareTables( + (left.String("TABLE_CAT") ?? left.SourceCatalog, left.String("TABLE_SCHEM"), + left.String("TABLE_NAME"), left.String("TABLE_TYPE")), + (right.String("TABLE_CAT") ?? right.SourceCatalog, right.String("TABLE_SCHEM"), + right.String("TABLE_NAME"), right.String("TABLE_TYPE")))); + return rows; + } + + internal static IEnumerable Catalogs(MetadataBatches result) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs)) + if (row.String("TABLE_CAT") is string catalog) yield return catalog; + yield break; + } + + foreach (var batch in result.Batches) + { + var catalogs = TryGetColumn(batch, "catalog"); + if (catalogs == null) continue; + for (int row = 0; row < batch.Length; row++) + if (String(catalogs, row) is string catalog) yield return catalog; + } + } + + internal static IEnumerable Schemas(MetadataBatches result) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas)) + if (row.String("TABLE_SCHEM") is string schema) + yield return new SchemaRow(row.String("TABLE_CATALOG") ?? row.SourceCatalog ?? "", schema); + yield break; + } + + foreach (var batch in result.Batches) + { + // Scoped SHOW SCHEMAS omits the catalog column. + var schemas = batch.Column(0) as StringArray; + var catalogs = result.SourceCatalog == null ? batch.Column(1) as StringArray : null; + if (schemas == null) continue; + for (int row = 0; row < batch.Length; row++) + { + string? schema = String(schemas, row); + if (schema != null) + yield return new SchemaRow(String(catalogs, row) ?? result.SourceCatalog ?? "", schema); + } + } + } + + internal static List Tables( + MetadataBatches result, IReadOnlyCollection? tableTypes = null, + bool normalizeEmptyTableType = true) + { + var rows = new List(); + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, tableTypes)) + { + string? schema = row.String("TABLE_SCHEM"); + string? table = row.String("TABLE_NAME"); + if (schema == null || table == null) continue; + rows.Add(new TableRow(row.String("TABLE_CAT") ?? row.SourceCatalog ?? "", schema, table, + DefaultTableType(row.String("TABLE_TYPE")), row.String("REMARKS") ?? "")); + } + return rows; + } + foreach (var batch in result.Batches) + { + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var types = TryGetColumn(batch, "tableType"); + var remarks = TryGetColumn(batch, "remarks"); + if (catalogs == null || schemas == null || tables == null) continue; + + for (int row = 0; row < batch.Length; row++) + { + string? rowCatalog = String(catalogs, row); + string? schema = String(schemas, row); + string? table = String(tables, row); + if (rowCatalog == null || schema == null || table == null) continue; + string? serverType = String(types, row); + string type = normalizeEmptyTableType + ? DefaultTableType(serverType) + : serverType ?? "TABLE"; + if (tableTypes != null && !tableTypes.Contains(type)) continue; + rows.Add(new TableRow(rowCatalog ?? "", schema, table, type, + String(remarks, row) ?? "")); + } + } + return rows; + } + + internal static IEnumerable Columns(IEnumerable results, + bool requireTableIdentifiers = false) + => ColumnGroups(results, requireTableIdentifiers).SelectMany(rows => rows); + + internal static IEnumerable> ColumnGroups(IEnumerable results, + bool requireTableIdentifiers = false) + { + var positions = new Dictionary(); + foreach (var result in results) + { + var group = new List(); + var rows = result.IsNative ? NativeColumns(result) + : result.Batches.SelectMany(batch => ShowColumns(batch, positions)); + foreach (var row in rows) + { + // GetObjects skips incomplete SHOW rows before assigning ordinals. + if (requireTableIdentifiers && (row.Catalog == null || row.Schema == null || + row.Table == null || string.IsNullOrEmpty(row.Name))) continue; + string key = $"{row.Catalog}.{row.Schema}.{row.Table}"; + positions.TryGetValue(key, out int position); + positions[key] = position + 1; + group.Add(row); + } + yield return group; + } + } + + private static IEnumerable NativeColumns(MetadataBatches result) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns)) + { + string? name = row.String("COLUMN_NAME"); + string? typeName = row.String("TYPE_NAME"); + if (name == null || typeName == null) continue; + // Both decoders expose zero-based ordinals; GetObjects adds its required offset. + yield return new ColumnRow(row.String("TABLE_CAT") ?? row.SourceCatalog, row.String("TABLE_SCHEM"), + row.String("TABLE_NAME"), name, typeName, row.Integer("NULLABLE") == 1, + checked((int)(row.Integer("ORDINAL_POSITION") ?? 0)), + row.String("COLUMN_DEF"), + string.Equals(row.String("IS_AUTO_INCREMENT"), "YES", StringComparison.OrdinalIgnoreCase)); + } + } + + private static IEnumerable ShowColumns(RecordBatch batch, Dictionary positions) + { + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var names = TryGetColumn(batch, "col_name"); + var types = TryGetColumn(batch, "columnType"); + var nullable = TryGetColumn(batch, "isNullable"); + if (names == null || types == null) yield break; + for (int row = 0; row < batch.Length; row++) + { + string? name = String(names, row); + string? typeName = String(types, row); + if (name == null || typeName == null) continue; + string? catalog = String(catalogs, row); + string? schema = String(schemas, row); + string? table = String(tables, row); + positions.TryGetValue($"{catalog}.{schema}.{table}", out int ordinal); + yield return new ColumnRow(catalog, schema, table, name, typeName, + !string.Equals(String(nullable, row), "false", StringComparison.OrdinalIgnoreCase), + ordinal, null, false); + } + } + + internal static IEnumerable PrimaryKeys( + MetadataBatches result, string catalog, string schema, string table) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys)) + if (row.String("COLUMN_NAME") is string column) + yield return new PrimaryKeyRow(row.String("TABLE_CAT") ?? catalog, + row.String("TABLE_SCHEM") ?? schema, row.String("TABLE_NAME") ?? table, + column, checked((int)(row.Integer("KEQ_SEQ") ?? 0)), row.String("PK_NAME") ?? ""); + yield break; + } + + int sequence = 0; + foreach (var batch in result.Batches) + { + var columns = TryGetColumn(batch, "col_name"); + var names = TryGetColumn(batch, "constraintName"); + var sequences = TryGetColumn(batch, "keySeq"); + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + if (columns == null) continue; + for (int row = 0; row < batch.Length; row++) + if (String(columns, row) is string column) + yield return new PrimaryKeyRow(String(catalogs, row) ?? catalog, + String(schemas, row) ?? schema, String(tables, row) ?? table, + column, Integer(sequences, row) ?? ++sequence, String(names, row) ?? ""); + } + } + + internal static IEnumerable ForeignKeys( + MetadataBatches result, string? parentCatalog, string? parentSchema, string? parentTable, + string catalog, string schema, string table) + { + if (result.IsNative) + { + foreach (var row in NativeRows(new[] { result }, + MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, + parentCatalog: parentCatalog, parentSchema: parentSchema, parentTable: parentTable)) + if (row.String("FKCOLUMN_NAME") is string column) + yield return new ForeignKeyRow(row.String("PKTABLE_CAT") ?? parentCatalog ?? "", + row.String("PKTABLE_SCHEM") ?? parentSchema ?? "", row.String("PKTABLE_NAME") ?? parentTable ?? "", + row.String("PKCOLUMN_NAME") ?? "", row.String("FKTABLE_CAT") ?? catalog, + row.String("FKTABLE_SCHEM") ?? schema, row.String("FKTABLE_NAME") ?? table, column, + checked((int)(row.Integer("KEQ_SEQ") ?? 0)), + checked((int)(row.Integer("UPDATE_RULE") ?? 0)), + checked((int)(row.Integer("DELETE_RULE") ?? 0)), row.String("FK_NAME") ?? "", + row.String("PK_NAME"), checked((int)(row.Integer("DEFERRABILITY") ?? 5))); + yield break; + } + + int sequence = 0; + foreach (var batch in result.Batches) + { + var parentCatalogs = TryGetColumn(batch, "parentCatalogName"); + var parentSchemas = TryGetColumn(batch, "parentNamespace"); + var parentTables = TryGetColumn(batch, "parentTableName"); + var parentColumns = TryGetColumn(batch, "parentColName"); + var catalogs = TryGetColumn(batch, "catalogName"); + var schemas = TryGetColumn(batch, "namespace"); + var tables = TryGetColumn(batch, "tableName"); + var columns = TryGetColumn(batch, "col_name"); + var names = TryGetColumn(batch, "constraintName"); + var sequences = TryGetColumn(batch, "keySeq"); + var updateRules = TryGetColumn(batch, "updateRule"); + var deleteRules = TryGetColumn(batch, "deleteRule"); + var deferrabilities = TryGetColumn(batch, "deferrability"); + if (columns == null) continue; + for (int row = 0; row < batch.Length; row++) + { + string? column = String(columns, row); + if (column == null) continue; + string? rowParentCatalog = String(parentCatalogs, row); + string? rowParentSchema = String(parentSchemas, row); + string? rowParentTable = String(parentTables, row); + // Match raw server identifiers before applying output fallbacks. + if (!MatchesParent(parentCatalog, parentSchema, parentTable, + rowParentCatalog, rowParentSchema, rowParentTable)) continue; + yield return new ForeignKeyRow(rowParentCatalog ?? parentCatalog ?? "", + rowParentSchema ?? parentSchema ?? "", rowParentTable ?? parentTable ?? "", + String(parentColumns, row) ?? "", String(catalogs, row) ?? catalog, + String(schemas, row) ?? schema, String(tables, row) ?? table, column, + Integer(sequences, row) ?? ++sequence, Integer(updateRules, row) ?? 0, + Integer(deleteRules, row) ?? 0, String(names, row) ?? "", null, + Integer(deferrabilities, row) ?? 5); + } + } + } + + internal static int CompareTables( + (string? Catalog, string? Schema, string? Table, string? TableType) left, + (string? Catalog, string? Schema, string? Table, string? TableType) right) + { + int order = StringComparer.Ordinal.Compare( + DefaultTableType(left.TableType), + DefaultTableType(right.TableType)); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.Catalog, right.Catalog); + if (order != 0) return order; + order = StringComparer.Ordinal.Compare(left.Schema, right.Schema); + return order != 0 ? order : StringComparer.Ordinal.Compare(left.Table, right.Table); + } + + internal static bool MatchesCatalog(CatalogFilter filter, string? actual) + => filter.Value == null || (actual != null && (filter.IsPattern + ? MatchesCatalogPattern(filter.Value, actual) + : string.Equals(filter.Value, actual, StringComparison.OrdinalIgnoreCase))); + + private static bool MatchesParent(string? catalog, string? schema, string? table, + string? actualCatalog, string? actualSchema, string? actualTable) + => MatchesCatalog(CatalogFilter.Exact(catalog), actualCatalog) && + MatchesCatalog(CatalogFilter.Exact(schema), actualSchema) && + MatchesCatalog(CatalogFilter.Exact(table), actualTable); + + internal static string DefaultTableType(string? value) => string.IsNullOrEmpty(value) ? "TABLE" : value!; + + private static string? String(StringArray? values, int row) + => values == null || values.IsNull(row) ? null : values.GetString(row); + + private static int? Integer(Int32Array? values, int row) + => values == null || values.IsNull(row) ? null : values.GetValue(row); + + private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray + { + try + { + return batch.Column(name) as T; + } + catch (ArgumentOutOfRangeException) + { + return null; + } + } + + private static string? LiteralCatalog(string? pattern) + { + if (pattern == null) return null; + var literal = new StringBuilder(); + bool escaped = false; + foreach (char character in pattern) + { + if (!escaped) + { + if (character is '%' or '_') return null; + if (character == '\\') + { + escaped = true; + continue; + } + } + literal.Append(character); + escaped = false; + } + if (escaped) literal.Append('\\'); + return literal.ToString(); + } + + private static bool MatchesCatalogPattern(string? pattern, string catalog) + { + if (pattern == null) return true; + var regex = new StringBuilder("^"); + bool escaped = false; + foreach (char character in pattern) + { + if (!escaped && character == '\\') + { + escaped = true; + continue; + } + regex.Append(!escaped && character == '%' ? ".*" + : !escaped && character == '_' ? "." + : Regex.Escape(character.ToString())); + escaped = false; + } + if (escaped) regex.Append(Regex.Escape("\\")); + regex.Append('$'); + return Regex.IsMatch(catalog, regex.ToString(), RegexOptions.CultureInvariant | RegexOptions.IgnoreCase, + TimeSpan.FromSeconds(1)); + } + } +} diff --git a/csharp/src/StatementExecution/NativeMetadataColumns.cs b/csharp/src/StatementExecution/NativeMetadataColumns.cs new file mode 100644 index 00000000..8ebc6394 --- /dev/null +++ b/csharp/src/StatementExecution/NativeMetadataColumns.cs @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using AdbcDrivers.HiveServer2; +using Apache.Arrow; +using Apache.Arrow.Types; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal sealed class NativeMetadataColumns + { + private readonly Dictionary _columns = new(StringComparer.Ordinal); + + internal NativeMetadataColumns(RecordBatch batch, Schema schema, MetadataOperation operation) + { + var fields = schema.FieldsList.Where(field => + operation != MetadataOperation.GetColumns || field.Name != "BASE_TYPE_NAME").ToList(); + int count = fields.Count; + if (batch.ColumnCount != count) + throw new DatabricksException($"Invalid native {operation} result: expected {count} columns, found {batch.ColumnCount}"); + + foreach (var expected in fields) + { + int index = batch.Schema.GetFieldIndex(expected.Name); + if (index < 0) + throw new DatabricksException($"Invalid native {operation} result: missing {expected.Name}"); + + IArrowArray array = batch.Column(index); + bool valid = expected.DataType.TypeId == ArrowTypeId.String + ? array is StringArray + : IsInteger(expected.DataType.TypeId) && IsInteger(batch.Schema.FieldsList[index].DataType.TypeId) + && array is Int8Array or Int16Array or Int32Array or Int64Array; + if (!valid || _columns.ContainsKey(expected.Name)) + throw new DatabricksException($"Invalid native {operation} result: unexpected type or duplicate {expected.Name}"); + _columns.Add(expected.Name, array); + } + } + + internal string? String(string name, int row) + { + var array = (StringArray)_columns[name]; + return array.IsNull(row) ? null : array.GetString(row); + } + + internal long? Integer(string name, int row) + { + var array = _columns[name]; + if (array.IsNull(row)) return null; + return array switch + { + Int8Array values => values.GetValue(row), + Int16Array values => values.GetValue(row), + Int32Array values => values.GetValue(row), + Int64Array values => values.GetValue(row), + _ => throw new DatabricksException($"Expected a native metadata integer, found {array.GetType().Name}") + }; + } + + private static bool IsInteger(ArrowTypeId type) => type is + ArrowTypeId.Int8 or ArrowTypeId.Int16 or ArrowTypeId.Int32 or ArrowTypeId.Int64; + } +} diff --git a/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs new file mode 100644 index 00000000..72571cb8 --- /dev/null +++ b/csharp/src/StatementExecution/NativeMetadataResultBuilder.cs @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Types; + +namespace AdbcDrivers.Databricks.StatementExecution +{ + internal static class NativeMetadataResultBuilder + { + internal static QueryResult Build( + MetadataBatches result, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + => Build(new[] { result }, schema, operation, tableTypes, parentCatalog, parentSchema, parentTable); + + internal static QueryResult Build( + IReadOnlyList results, Schema schema, MetadataOperation operation, + IReadOnlyCollection? tableTypes = null, + string? parentCatalog = null, string? parentSchema = null, string? parentTable = null) + { + var rows = MetadataRowReader.NativeRows( + results, schema, operation, tableTypes, parentCatalog, parentSchema, parentTable); + var arrays = new List(schema.FieldsList.Count); + foreach (var field in schema.FieldsList) + { + switch (field.DataType.TypeId) + { + case ArrowTypeId.String: + var strings = new StringArray.Builder(); + foreach (var row in rows) + { + string? value; + if (operation == MetadataOperation.GetColumns && field.Name == "BASE_TYPE_NAME") + { + string? typeName = row.String("TYPE_NAME"); + value = typeName == null ? null : ColumnMetadataHelper.GetBaseTypeName(typeName); + } + else + value = row.String(field.Name); + if ((field.Name == "TABLE_CATALOG" && operation == MetadataOperation.GetSchemas) || + (field.Name == "TABLE_CAT" && operation is MetadataOperation.GetTables or MetadataOperation.GetColumns)) + value ??= row.SourceCatalog ?? ""; + if (field.Name == "TABLE_TYPE" && operation == MetadataOperation.GetTables) + value = MetadataRowReader.DefaultTableType(value); + if (value == null) strings.AppendNull(); else strings.Append(value); + } + arrays.Add(strings.Build()); + break; + + case ArrowTypeId.Int8: + var int8 = new Int8Array.Builder(); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int8.Append(value.HasValue ? checked((sbyte)value.Value) : (sbyte?)null), + () => int8.Build())); + break; + + case ArrowTypeId.Int16: + var int16 = new Int16Array.Builder(); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int16.Append(value.HasValue ? checked((short)value.Value) : (short?)null), + () => int16.Build())); + break; + + case ArrowTypeId.Int32: + var int32 = new Int32Array.Builder(); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int32.Append(value.HasValue ? checked((int)value.Value) : (int?)null), + () => int32.Build())); + break; + + case ArrowTypeId.Int64: + var int64 = new Int64Array.Builder(); + arrays.Add(BuildIntegers(rows, field.Name, operation, + value => int64.Append(value), () => int64.Build())); + break; + + default: + throw new DatabricksException($"Unsupported native metadata field type: {field.DataType}"); + } + } + + return new QueryResult(rows.Count, new HiveInfoArrowStream(schema, arrays.ToArray())); + } + + private static IArrowArray BuildIntegers( + IReadOnlyList rows, string field, MetadataOperation operation, + Action append, Func build) + { + foreach (var row in rows) + { + long? value = row.Integer(field); + if (operation == MetadataOperation.GetColumns && (field is "COLUMN_SIZE" or "DECIMAL_DIGITS")) + { + var precisionAndScale = ColumnMetadataHelper.NormalizePrecisionAndScale( + row.String("TYPE_NAME"), row.Integer("COLUMN_SIZE"), row.Integer("DECIMAL_DIGITS")); + value = field == "COLUMN_SIZE" ? precisionAndScale.ColumnSize : precisionAndScale.DecimalDigits; + } + append(value); + } + return build(); + } + } +} diff --git a/csharp/src/StatementExecution/StatementExecutionClient.cs b/csharp/src/StatementExecution/StatementExecutionClient.cs index b953fed7..96ece3c5 100644 --- a/csharp/src/StatementExecution/StatementExecutionClient.cs +++ b/csharp/src/StatementExecution/StatementExecutionClient.cs @@ -227,6 +227,21 @@ public async Task ExecuteStatementAsync( httpRequest.Headers.TryAddWithoutValidation("x-databricks-sea-can-run-fully-sync", "true"); } + if (request.IsMetadata && request.MetadataOperation is MetadataOperation operation) + { + httpRequest.Headers.TryAddWithoutValidation("x-databricks-metadata-operation-type", operation switch + { + MetadataOperation.GetCatalogs => "GetCatalogs", + MetadataOperation.GetSchemas => "GetSchemas", + MetadataOperation.GetTables => "GetTables", + MetadataOperation.GetColumns => "GetColumns", + MetadataOperation.GetPrimaryKeys => "GetPrimaryKeys", + MetadataOperation.GetCrossReference => "GetCrossReference", + _ => throw new ArgumentOutOfRangeException(nameof(operation), operation, null), + }); + httpRequest.Headers.TryAddWithoutValidation("x-databricks-require-thrift-native-metadata", "true"); + } + var response = await _httpClient.SendAsync(httpRequest, cancellationToken).ConfigureAwait(false); await EnsureSuccessStatusCodeAsync(response).ConfigureAwait(false); diff --git a/csharp/src/StatementExecution/StatementExecutionConnection.cs b/csharp/src/StatementExecution/StatementExecutionConnection.cs index ce046f8c..4add31a4 100644 --- a/csharp/src/StatementExecution/StatementExecutionConnection.cs +++ b/csharp/src/StatementExecution/StatementExecutionConnection.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Net.Http; using System.Threading; @@ -710,13 +711,15 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string using var cts = CreateMetadataTimeoutCts(); // Pass catalog through with SPARK→null normalization, matching Thrift - // which sends catalog as-is to the server. ExecuteShowColumnsAsync + // which sends catalog as-is to the server. ReadColumnsAsync // handles null by iterating all catalogs. string? resolvedCatalog = DatabricksConnection.HandleSparkCatalog(catalog); - List batches; + ColumnMetadataResult columns; try { - batches = ExecuteShowColumnsAsync(resolvedCatalog, dbSchema, tableName, null, cts.Token) + columns = ReadColumnsAsync( + resolvedCatalog, dbSchema, tableName, null, cts.Token, + catalogFilter: CatalogFilter.Exact(resolvedCatalog)) .GetAwaiter().GetResult(); } catch (Exception ex) when (cts.IsCancellationRequested && ex is not TimeoutException) @@ -728,27 +731,11 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string } var fields = new List(); - foreach (var batch in batches) + foreach (var column in columns.Rows) { - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); - - if (colNameArray == null || columnTypeArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - - string colName = colNameArray.GetString(i); - string colType = columnTypeArray.GetString(i); - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - short typeCode = ColumnMetadataHelper.GetDataTypeCode(colType); - IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, colType, false, null, null); - fields.Add(new Field(colName, arrowType, nullable)); - } + short typeCode = ColumnMetadataHelper.GetDataTypeCode(column.TypeName); + IArrowType arrowType = HiveServer2Connection.GetArrowType(typeCode, column.TypeName, false, null, null); + fields.Add(new Field(column.Name, arrowType, column.Nullable)); } activity?.SetTag("result_fields", fields.Count); @@ -761,80 +748,21 @@ public override Schema GetTableSchema(string? catalog, string? dbSchema, string async Task> IGetObjectsDataProvider.GetCatalogsAsync(string? catalogPattern, CancellationToken cancellationToken) { string sql = new ShowCatalogsCommand(catalogPattern).Build(); - List batches; - try - { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - // Object-not-found → empty (GetObjects nested-shape path; matches the - // flat metadata methods and the JDBC reference driver). - return System.Array.Empty(); - } - var result = new List(); - foreach (var batch in batches) - { - var catalogArray = TryGetColumn(batch, "catalog"); - if (catalogArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (!catalogArray.IsNull(i)) - result.Add(catalogArray.GetString(i)); - } - } - return result; + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetCatalogs, cancellationToken, + catalogFilter: CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Catalogs(result).ToList(); } async Task> IGetObjectsDataProvider.GetSchemasAsync(string? catalogPattern, string? schemaPattern, CancellationToken cancellationToken) { - // Note: catalogPattern comes from GetObjectsResultBuilder which resolves individual - // catalog names before calling this method. Despite the "pattern" name (from the - // IGetObjectsDataProvider interface), the value passed to ShowSchemasCommand is used - // as a literal catalog identifier (backtick-quoted), not a wildcard pattern. string sql = new ShowSchemasCommand(catalogPattern, schemaPattern).Build(); - List batches; - try - { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return System.Array.Empty<(string, string)>(); - } - - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showSchemasInAllCatalogs = catalogPattern == null; - - var result = new List<(string, string)>(); - foreach (var batch in batches) - { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (showSchemasInAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - string catalog = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalogPattern ?? ""; - result.Add((catalog, schemaArray.GetString(i))); - } - } - return result; + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetSchemas, cancellationToken, catalogPattern, + CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Schemas(result) + .Select(row => (row.Catalog, row.Schema)).ToList(); } async Task> IGetObjectsDataProvider.GetTablesAsync( @@ -852,48 +780,11 @@ async Task> IGetObjectsDataProvider.GetCatalogsAsync(strin string sql = new ShowTablesCommand(catalogPattern, schemaPattern, tableNamePattern).Build(); - List batches; - try - { - batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return System.Array.Empty<(string, string, string, string)>(); - } - var result = new List<(string, string, string, string)>(); - foreach (var batch in batches) - { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var tableTypeArray = TryGetColumn(batch, "tableType"); - - if (catalogArray == null || schemaArray == null || tableArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; - - string tableType = "TABLE"; - if (tableTypeArray != null && !tableTypeArray.IsNull(i)) - { - string serverType = tableTypeArray.GetString(i); - if (!string.IsNullOrEmpty(serverType)) - tableType = serverType; - } - - // Empty (non-null) tableTypes filters to NO types (zero rows), - // matching databricks-jdbc. A null tableTypes means "all types" - // and skips this filter entirely. See METADATA-035. - if (tableTypes != null && !tableTypes.Contains(tableType)) - continue; - - result.Add((catalogArray.GetString(i), schemaArray.GetString(i), - tableArray.GetString(i), tableType)); - } - } - return result; + var result = await ReadMetadataAsync( + sql, MetadataOperation.GetTables, cancellationToken, catalogPattern, + CatalogFilter.Pattern(catalogPattern)).ConfigureAwait(false); + return MetadataRowReader.Tables(result, tableTypes) + .Select(row => (row.Catalog, row.Schema, row.Table, row.TableType)).ToList(); } async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPattern, string? schemaPattern, @@ -901,84 +792,39 @@ async Task IGetObjectsDataProvider.PopulateColumnInfoAsync(string? catalogPatter Dictionary>> catalogMap, CancellationToken cancellationToken) { - List batches; - try - { - batches = await ExecuteShowColumnsAsync(catalogPattern, schemaPattern, tablePattern, columnPattern, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - // Object-not-found → leave the column info unpopulated (empty), matching - // the flat metadata methods and the JDBC reference driver. - return; - } - - var tablePositions = new Dictionary(); - - foreach (var batch in batches) + var columns = await ReadMetadataOrEmptyAsync(MetadataOperation.GetColumns, + () => ReadColumnsAsync(catalogPattern, schemaPattern, tablePattern, + columnPattern, cancellationToken, CatalogFilter.Pattern(catalogPattern), + requireTableIdentifiers: true), + () => new ColumnMetadataResult(System.Array.Empty())).ConfigureAwait(false); + foreach (var column in columns.Rows) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableNameArray = TryGetColumn(batch, "tableName"); - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); - - if (catalogArray == null || schemaArray == null || tableNameArray == null || - colNameArray == null || columnTypeArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableNameArray.IsNull(i) || - colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - - string cat = catalogArray.GetString(i); - string sch = schemaArray.GetString(i); - string tbl = tableNameArray.GetString(i); - string colName = colNameArray.GetString(i); - string colType = columnTypeArray.GetString(i); - - if (string.IsNullOrEmpty(colName)) continue; - - string tableKey = $"{cat}.{sch}.{tbl}"; - if (!tablePositions.ContainsKey(tableKey)) - tablePositions[tableKey] = 1; - int position = tablePositions[tableKey]++; - - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - if (catalogMap.TryGetValue(cat, out var schemaMap) - && schemaMap.TryGetValue(sch, out var tableMap) - && tableMap.TryGetValue(tbl, out var tableInfo)) - { - ColumnMetadataHelper.PopulateTableInfoFromTypeName( - tableInfo, colName, colType, position, nullable); - } - } + if (catalogMap.TryGetValue(column.Catalog!, out var schemaMap) + && schemaMap.TryGetValue(column.Schema!, out var tableMap) + && tableMap.TryGetValue(column.Table!, out var tableInfo)) + ColumnMetadataHelper.PopulateTableInfoFromTypeName( + tableInfo, column.Name, column.TypeName, checked(column.Ordinal + 1), column.Nullable, + columnDefault: column.Default, isAutoIncrement: column.IsAutoIncrement); } } - private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray + internal async Task> ExecuteMetadataSqlAsync(string sql, CancellationToken cancellationToken = default) { - try - { - return batch.Column(name) as T; - } - catch (ArgumentOutOfRangeException) - { - return null; - } + var result = await ExecuteMetadataCommandAsync(sql, null, cancellationToken).ConfigureAwait(false); + return result.Batches; } - internal async Task> ExecuteMetadataSqlAsync(string sql, CancellationToken cancellationToken = default) + internal async Task ExecuteMetadataCommandAsync( + string sql, MetadataOperation? operation, CancellationToken cancellationToken, + string? sourceCatalog = null, CatalogFilter catalogFilter = default) { var batches = new List(); using var stmt = (StatementExecutionStatement)CreateStatement(); stmt.SqlQuery = sql; - var result = await stmt.ExecuteQueryAsync(cancellationToken, isMetadataExecution: true).ConfigureAwait(false); + var result = await stmt.ExecuteQueryAsync( + cancellationToken, isMetadataExecution: true, metadataOperation: operation).ConfigureAwait(false); using var stream = result.Stream; - if (stream == null) return batches; + if (stream == null) return new MetadataBatches(batches, stmt.IsNativeMetadataResult, sourceCatalog, catalogFilter); while (true) { cancellationToken.ThrowIfCancellationRequested(); @@ -986,7 +832,41 @@ internal async Task> ExecuteMetadataSqlAsync(string sql, Cance if (batch == null) break; batches.Add(batch); } - return batches; + return new MetadataBatches(batches, stmt.IsNativeMetadataResult, sourceCatalog, catalogFilter); + } + + internal Task ReadMetadataAsync( + string sql, MetadataOperation operation, CancellationToken cancellationToken, + string? sourceCatalog = null, CatalogFilter catalogFilter = default) + { + Activity.Current?.SetTag("sql_query", sql); + return ReadMetadataOrEmptyAsync(operation, + () => ExecuteMetadataCommandAsync(sql, operation, cancellationToken, sourceCatalog, catalogFilter), + () => new MetadataBatches(new List(), false, sourceCatalog, catalogFilter)); + } + + internal static async Task ReadMetadataOrEmptyAsync( + MetadataOperation operation, Func> read, Func empty) + { + try + { + return await read().ConfigureAwait(false); + } + catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) + { + string name = operation switch + { + MetadataOperation.GetCatalogs => "get_catalogs", + MetadataOperation.GetSchemas => "get_schemas", + MetadataOperation.GetTables => "get_tables", + MetadataOperation.GetColumns => "get_columns", + MetadataOperation.GetPrimaryKeys => "get_primary_keys", + MetadataOperation.GetCrossReference => "get_cross_reference", + _ => operation.ToString(), + }; + Activity.Current?.AddEvent($"statement.{name}.object_not_found", [new("error", ex.Message)]); + return empty(); + } } internal List ExecuteMetadataSql(string sql, CancellationToken cancellationToken = default) @@ -994,47 +874,44 @@ internal List ExecuteMetadataSql(string sql, CancellationToken canc return ExecuteMetadataSqlAsync(sql, cancellationToken).GetAwaiter().GetResult(); } - /// - /// Executes a SHOW COLUMNS command. When catalog is null, iterates over all catalogs - /// since SHOW COLUMNS IN ALL CATALOGS is not yet supported by the backend. - /// - internal async Task> ExecuteShowColumnsAsync( + // SHOW COLUMNS IN ALL CATALOGS is unavailable, so fan out over catalogs. + internal async Task ReadColumnsAsync( string? catalog, string? schemaPattern, string? tablePattern, string? columnPattern, - CancellationToken cancellationToken) + CancellationToken cancellationToken, CatalogFilter catalogFilter = default, + bool requireTableIdentifiers = false) { + var results = new List(); if (catalog != null) { string sql = new ShowColumnsCommand(catalog, schemaPattern, tablePattern, columnPattern).Build(); - return await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var result = await ExecuteMetadataCommandAsync( + sql, MetadataOperation.GetColumns, cancellationToken, catalog, catalogFilter).ConfigureAwait(false); + results.Add(result); + return new ColumnMetadataResult(results, requireTableIdentifiers); } - // SHOW COLUMNS IN ALL CATALOGS is not supported — iterate over each catalog. - // TODO: Remove this fallback when the backend supports SHOW COLUMNS IN ALL CATALOGS. - var allBatches = new List(); string catalogsSql = new ShowCatalogsCommand(null).Build(); - var catalogBatches = await ExecuteMetadataSqlAsync(catalogsSql, cancellationToken).ConfigureAwait(false); - - foreach (var batch in catalogBatches) + var catalogs = await ExecuteMetadataCommandAsync( + catalogsSql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); + foreach (string sourceCatalog in MetadataRowReader.Catalogs(catalogs)) { - var catalogArray = batch.Column(0) as StringArray; - if (catalogArray == null) continue; - for (int i = 0; i < catalogArray.Length; i++) + string sql = new ShowColumnsCommand( + sourceCatalog, schemaPattern, tablePattern, columnPattern).Build(); + MetadataBatches result; + try { - if (catalogArray.IsNull(i)) continue; - string cat = catalogArray.GetString(i); - string sql = new ShowColumnsCommand(cat, schemaPattern, tablePattern, columnPattern).Build(); - try - { - var batches = await ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - allBatches.AddRange(batches); - } - catch - { - // Skip catalogs we can't access (permission errors) - } + result = await ExecuteMetadataCommandAsync( + sql, MetadataOperation.GetColumns, cancellationToken, + sourceCatalog, CatalogFilter.Exact(sourceCatalog)).ConfigureAwait(false); + } + catch + { + // Skip catalogs we can't access (permission errors) + continue; } + results.Add(result); } - return allBatches; + return new ColumnMetadataResult(results, requireTableIdentifiers); } internal bool EnablePKFK => _enablePKFK; diff --git a/csharp/src/StatementExecution/StatementExecutionModels.cs b/csharp/src/StatementExecution/StatementExecutionModels.cs index a2276a7a..f812fe34 100644 --- a/csharp/src/StatementExecution/StatementExecutionModels.cs +++ b/csharp/src/StatementExecution/StatementExecutionModels.cs @@ -19,6 +19,16 @@ namespace AdbcDrivers.Databricks.StatementExecution { + internal enum MetadataOperation + { + GetCatalogs, + GetSchemas, + GetTables, + GetColumns, + GetPrimaryKeys, + GetCrossReference, + } + // ============================================================================ // Session Management Models // ============================================================================ @@ -185,6 +195,9 @@ public class ExecuteStatementRequest [JsonIgnore] public bool IsMetadata { get; set; } + [JsonIgnore] + internal MetadataOperation? MetadataOperation { get; set; } + /// /// Maximum number of rows to return. /// @@ -400,6 +413,9 @@ public class ResultManifest /// [JsonPropertyName("is_volume_operation")] public bool? IsVolumeOperation { get; set; } + + [JsonPropertyName("is_native_metadata_result")] + public bool? IsNativeMetadataResult { get; set; } } /// diff --git a/csharp/src/StatementExecution/StatementExecutionStatement.cs b/csharp/src/StatementExecution/StatementExecutionStatement.cs index 10542a3a..5b070f9b 100644 --- a/csharp/src/StatementExecution/StatementExecutionStatement.cs +++ b/csharp/src/StatementExecution/StatementExecutionStatement.cs @@ -79,6 +79,7 @@ internal class StatementExecutionStatement : TracingStatement // in-hand and the statement is closed server-side, so Dispose must NOT re-close it. private bool _statementClosedByServer; private string? _sqlQuery; + internal bool IsNativeMetadataResult { get; private set; } // Marks a driver-internal statement (e.g. the USE CATALOG issued by EnsureCatalogScopedAsync). // Mirrors DatabricksStatement.IsInternalCall on Thrift: it prevents catalog-scoping recursion @@ -305,10 +306,15 @@ public override QueryResult ExecuteQuery() /// Cancellation token. /// When true, adds the x-databricks-sea-can-run-fully-sync /// header for optimized metadata query execution on the server. - public async Task ExecuteQueryAsync( + public Task ExecuteQueryAsync( CancellationToken cancellationToken = default, bool isMetadataExecution = false) + => ExecuteQueryAsync(cancellationToken, isMetadataExecution, null); + + internal async Task ExecuteQueryAsync( + CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation) { + IsNativeMetadataResult = false; if (_isMetadataCommand) { return await ExecuteMetadataCommandAsync(cancellationToken).ConfigureAwait(false); @@ -324,7 +330,7 @@ public async Task ExecuteQueryAsync( lock (_cancelLock) { _executeCts = cts; } try { - return await ExecuteQueryInternalAsync(cts.Token, isMetadataExecution).ConfigureAwait(false); + return await ExecuteQueryInternalAsync(cts.Token, isMetadataExecution, metadataOperation).ConfigureAwait(false); } catch (OperationCanceledException ex) when (!cancellationToken.IsCancellationRequested) { @@ -386,7 +392,8 @@ private async Task EnsureCatalogScopedAsync(CancellationToken cancellationToken) _connection.UpdateCurrentCatalog(catalog); } - private async Task ExecuteQueryInternalAsync(CancellationToken cancellationToken, bool isMetadataExecution) + private async Task ExecuteQueryInternalAsync( + CancellationToken cancellationToken, bool isMetadataExecution, MetadataOperation? metadataOperation = null) { // If the caller explicitly scoped this statement to a catalog, set the session's // current catalog first via USE CATALOG so a 2-level `schema`.`table` name @@ -410,11 +417,13 @@ private async Task ExecuteQueryInternalAsync(CancellationToken canc WaitTimeout = _waitTimeout, OnWaitTimeout = "CONTINUE", IsMetadata = isMetadataExecution, + MetadataOperation = metadataOperation, QueryTags = ParseQueryTags(_queryTags) }; // Execute the statement var response = await _client.ExecuteStatementAsync(request, cancellationToken).ConfigureAwait(false); + bool? initialNativeMetadata = response.Manifest?.IsNativeMetadataResult; _currentStatementId = response.StatementId; // Reset per-execution: statements are reusable, so this must reflect the CURRENT // execution's state, not a prior CLOSED result. Otherwise a later SUCCEEDED (genuinely @@ -462,6 +471,12 @@ private async Task ExecuteQueryInternalAsync(CancellationToken canc _statementClosedByServer = true; } + // The server may fall back to SHOW even with the require-native header. + // Only the response manifest selects the decoder. + IsNativeMetadataResult = response.Manifest?.IsNativeMetadataResult + ?? initialNativeMetadata + ?? false; + // Check for truncated results warning if (response.Manifest?.Truncated == true) { @@ -1086,7 +1101,7 @@ private static async Task>> FetchAllChunkSegmentsAsync /// catalog for SHOW commands when not querying all catalogs). /// // TODO: Once the backend supports SHOW COLUMNS IN ALL CATALOGS, the - // ExecuteShowColumnsAsync iterate-all-catalogs fallback can be removed. + // ReadColumnsAsync catalog fanout can be removed. private string? EffectiveCatalog { get @@ -1278,22 +1293,19 @@ private async Task GetCatalogsAsync(CancellationToken cancellationT // matching Thrift behavior (Thrift RPC has no catalog filter for GetCatalogs). string sql = new ShowCatalogsCommand(null).Build(); activity?.SetTag("sql_query", sql); - var batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); + var result = await _connection.ExecuteMetadataCommandAsync( + sql, MetadataOperation.GetCatalogs, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateCatalogsSchema(), MetadataOperation.GetCatalogs); var tableCatBuilder = new StringArray.Builder(); int count = 0; - foreach (var batch in batches) + foreach (string catalog in MetadataRowReader.Catalogs(result)) { - var catalogArray = TryGetColumn(batch, "catalog"); - if (catalogArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (!catalogArray.IsNull(i)) - { - tableCatBuilder.Append(catalogArray.GetString(i)); - count++; - } - } + tableCatBuilder.Append(catalog); + count++; } activity?.SetTag("result_count", count); @@ -1321,57 +1333,22 @@ private async Task GetSchemasAsync(CancellationToken cancellationTo EscapePatternWildcardsInName(_metadataSchemaName)).Build(); activity?.SetTag("sql_query", sql); - // Object-not-found (missing catalog/schema/table, or the server rejecting - // an empty/invalid name) → return an EMPTY result, matching both the Thrift - // path and the JDBC reference driver (isObjectNotFoundException). Restores - // the #388 behavior; the "make SEA throw" premise was disproven — Thrift - // returns empty on object-not-found too. - List batches; - try - { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_schemas.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptySchemasResult(); - } - - // SHOW SCHEMAS IN ALL CATALOGS returns 2 columns: databaseName, catalog - // SHOW SCHEMAS IN `catalog` returns 1 column: databaseName - bool showAllCatalogs = catalog == null; + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetSchemas, cancellationToken, catalog, + CatalogFilter.Exact(_escapePatternWildcards ? catalog : null)).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateSchemasSchema(), MetadataOperation.GetSchemas); var tableSchemaBuilder = new StringArray.Builder(); var tableCatalogBuilder = new StringArray.Builder(); int count = 0; - foreach (var batch in batches) + foreach (var row in MetadataRowReader.Schemas(result)) { - StringArray? catalogArray = null; - StringArray? schemaArray = null; - - if (showAllCatalogs) - { - schemaArray = batch.Column(0) as StringArray; - catalogArray = batch.Column(1) as StringArray; - } - else - { - schemaArray = batch.Column(0) as StringArray; - } - - if (schemaArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (schemaArray.IsNull(i)) continue; - tableSchemaBuilder.Append(schemaArray.GetString(i)); - string catalogValue = catalogArray != null && !catalogArray.IsNull(i) - ? catalogArray.GetString(i) - : catalog ?? ""; - tableCatalogBuilder.Append(catalogValue); - count++; - } + tableSchemaBuilder.Append(row.Schema); + tableCatalogBuilder.Append(row.Catalog); + count++; } activity?.SetTag("result_count", count); @@ -1410,29 +1387,11 @@ private async Task GetTablesAsync(CancellationToken cancellationTok EscapePatternWildcardsInName(_metadataTableName)).Build(); activity?.SetTag("sql_query", sql); - List batches; - try - { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_tables.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptyTablesResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetTables, cancellationToken, catalog, + _escapePatternWildcards ? CatalogFilter.Exact(catalog) : CatalogFilter.Pattern(catalog)).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; - var tableCatBuilder = new StringArray.Builder(); - var tableSchemaBuilder = new StringArray.Builder(); - var tableNameBuilder = new StringArray.Builder(); - var tableTypeBuilder = new StringArray.Builder(); - var remarksBuilder = new StringArray.Builder(); - var typeCatBuilder = new StringArray.Builder(); - var typeSchemaBuilder = new StringArray.Builder(); - var typeNameBuilder = new StringArray.Builder(); - var selfRefColBuilder = new StringArray.Builder(); - var refGenBuilder = new StringArray.Builder(); // Issue #526: match JDBC's MetadataResultSetBuilder and the Thrift path - // the types filter is a case-SENSITIVE exact match against the server's // uppercase type names (TABLE/VIEW/...). Use Ordinal, not OrdinalIgnoreCase. @@ -1444,33 +1403,36 @@ private async Task GetTablesAsync(CancellationToken cancellationTok StringComparer.Ordinal) : null; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateTablesSchema(), MetadataOperation.GetTables, + tableTypes: tableTypeFilter); + + var tableCatBuilder = new StringArray.Builder(); + var tableSchemaBuilder = new StringArray.Builder(); + var tableNameBuilder = new StringArray.Builder(); + var tableTypeBuilder = new StringArray.Builder(); + var remarksBuilder = new StringArray.Builder(); + var typeCatBuilder = new StringArray.Builder(); + var typeSchemaBuilder = new StringArray.Builder(); + var typeNameBuilder = new StringArray.Builder(); + var selfRefColBuilder = new StringArray.Builder(); + var refGenBuilder = new StringArray.Builder(); int count = 0; - foreach (var batch in batches) + foreach (var row in MetadataRowReader.Tables( + result, tableTypeFilter, normalizeEmptyTableType: false)) { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var tableTypeArray = TryGetColumn(batch, "tableType"); - var remarksArray = TryGetColumn(batch, "remarks"); - if (catalogArray == null || schemaArray == null || tableArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i)) continue; - string tableType = tableTypeArray != null && !tableTypeArray.IsNull(i) ? tableTypeArray.GetString(i) : "TABLE"; - if (tableTypeFilter != null && !tableTypeFilter.Contains(tableType)) continue; - tableCatBuilder.Append(catalogArray.GetString(i)); - tableSchemaBuilder.Append(schemaArray.GetString(i)); - tableNameBuilder.Append(tableArray.GetString(i)); - tableTypeBuilder.Append(tableType); - remarksBuilder.Append(remarksArray != null && !remarksArray.IsNull(i) ? remarksArray.GetString(i) : ""); - typeCatBuilder.AppendNull(); - typeSchemaBuilder.AppendNull(); - typeNameBuilder.AppendNull(); - selfRefColBuilder.AppendNull(); - refGenBuilder.AppendNull(); - count++; - } + tableCatBuilder.Append(row.Catalog); + tableSchemaBuilder.Append(row.Schema); + tableNameBuilder.Append(row.Table); + tableTypeBuilder.Append(row.TableType); + remarksBuilder.Append(row.Remarks); + typeCatBuilder.AppendNull(); + typeSchemaBuilder.AppendNull(); + typeNameBuilder.AppendNull(); + selfRefColBuilder.AppendNull(); + refGenBuilder.AppendNull(); + count++; } activity?.SetTag("result_count", count); @@ -1504,67 +1466,24 @@ private async Task GetColumnsAsync(CancellationToken cancellationTo return FlatColumnsResultBuilder.BuildFlatColumnsResult( System.Array.Empty<(string, string, string, TableInfo)>()); - List batches; - try - { - batches = await _connection.ExecuteShowColumnsAsync( + var columns = await StatementExecutionConnection.ReadMetadataOrEmptyAsync( + MetadataOperation.GetColumns, + () => _connection.ReadColumnsAsync( catalog, EscapePatternWildcardsInName(_metadataSchemaName), EscapePatternWildcardsInName(_metadataTableName), EscapePatternWildcardsInName(_metadataColumnName), - cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_columns.object_not_found", [ - new("error", ex.Message) - ]); - return FlatColumnsResultBuilder.BuildFlatColumnsResult( - System.Array.Empty<(string, string, string, TableInfo)>()); - } - - var tableInfos = new Dictionary(); - - foreach (var batch in batches) - { - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - var colNameArray = TryGetColumn(batch, "col_name"); - var columnTypeArray = TryGetColumn(batch, "columnType"); - var isNullableArray = TryGetColumn(batch, "isNullable"); - - if (catalogArray == null || schemaArray == null || tableArray == null || - colNameArray == null || columnTypeArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (catalogArray.IsNull(i) || schemaArray.IsNull(i) || tableArray.IsNull(i) || - colNameArray.IsNull(i) || columnTypeArray.IsNull(i)) continue; - - string cat = catalogArray.GetString(i); - string sch = schemaArray.GetString(i); - string tbl = tableArray.GetString(i); - string key = $"{cat}.{sch}.{tbl}"; - - if (!tableInfos.ContainsKey(key)) - tableInfos[key] = (cat, sch, tbl, new TableInfo("TABLE")); - - var entry = tableInfos[key]; - bool nullable = isNullableArray == null || isNullableArray.IsNull(i) || - !isNullableArray.GetString(i).Equals("false", StringComparison.OrdinalIgnoreCase); - - ColumnMetadataHelper.PopulateTableInfoFromTypeName( - entry.info, - colNameArray.GetString(i), - columnTypeArray.GetString(i), - entry.info.ColumnName.Count, - nullable); - } - } - - activity?.SetTag("result_tables", tableInfos.Count); - return FlatColumnsResultBuilder.BuildFlatColumnsResult(tableInfos.Values); + cancellationToken, CatalogFilter.Exact(_escapePatternWildcards ? catalog : null)), + () => new ColumnMetadataResult(System.Array.Empty())).ConfigureAwait(false); + + IsNativeMetadataResult = columns.IsNative; + if (!columns.IsNative) + activity?.SetTag("result_tables", columns.Rows + .Where(column => column.Catalog != null && column.Schema != null && column.Table != null) + .Select(column => $"{column.Catalog}.{column.Schema}.{column.Table}") + .Distinct().Count()); + return await FlatColumnsResultBuilder.BuildFlatColumnsResultAsync( + columns, cancellationToken).ConfigureAwait(false); }, "GetColumns").ConfigureAwait(false); } @@ -1801,49 +1720,15 @@ private async Task GetPrimaryKeysAsyncNoThrow( string sql = new ShowKeysCommand(catalog!, schema!, table!).Build(); activity?.SetTag("sql_query", sql); - List batches; - try - { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - activity?.AddEvent("statement.get_primary_keys.object_not_found", [ - new("error", ex.Message) - ]); - return MetadataSchemaFactory.CreateEmptyPrimaryKeysResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetPrimaryKeys, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreatePrimaryKeysSchema(), MetadataOperation.GetPrimaryKeys); - var keys = new List<(string, string, string, string, int, string)>(); - int seq = 0; - foreach (var batch in batches) - { - var colNameArray = TryGetColumn(batch, "col_name"); - var keyNameArray = TryGetColumn(batch, "constraintName"); - var keySeqArray = TryGetColumn(batch, "keySeq"); - // Read the identifier columns back from the SHOW KEYS response so - // TABLE_CAT/TABLE_SCHEM/TABLE_NAME reflect the server's stored (canonical) - // casing rather than echoing the caller's input case. Mirrors the JDBC - // reference driver (PRIMARY_KEYS_COLUMNS maps TABLE_CAT→catalogName, - // TABLE_SCHEM→namespace, TABLE_NAME→tableName) and this driver's own - // FetchCrossReferenceAsync. Fall back to the input args when the server - // omits a value, so behavior is never worse than before. - var catalogArray = TryGetColumn(batch, "catalogName"); - var schemaArray = TryGetColumn(batch, "namespace"); - var tableArray = TryGetColumn(batch, "tableName"); - if (colNameArray == null) continue; - for (int i = 0; i < batch.Length; i++) - { - if (colNameArray.IsNull(i)) continue; - int keySeq = keySeqArray != null && !keySeqArray.IsNull(i) ? keySeqArray.GetValue(i)!.Value : ++seq; - string pkName = keyNameArray != null && !keyNameArray.IsNull(i) ? keyNameArray.GetString(i) : ""; - string rowCatalog = catalogArray != null && !catalogArray.IsNull(i) ? catalogArray.GetString(i) : catalog!; - string rowSchema = schemaArray != null && !schemaArray.IsNull(i) ? schemaArray.GetString(i) : schema!; - string rowTable = tableArray != null && !tableArray.IsNull(i) ? tableArray.GetString(i) : table!; - keys.Add((rowCatalog, rowSchema, rowTable, - colNameArray.GetString(i), keySeq, pkName)); - } - } + var keys = MetadataRowReader.PrimaryKeys(result, catalog!, schema!, table!) + .Select(row => (row.Catalog, row.Schema, row.Table, row.Column, row.Sequence, row.Name)).ToList(); activity?.SetTag("result_count", keys.Count); return MetadataSchemaFactory.BuildPrimaryKeysResult(keys); @@ -1876,7 +1761,7 @@ private async Task GetCrossReferenceAsync(CancellationToken cancell // Pass the parent catalog to the filter ONLY when the caller explicitly supplied it. When // unset, _metadataCatalogName holds the connection's seeded default catalog; using it as the - // parent-catalog filter (added in the ParentMatches step) wrongly drops every FK whose real + // parent-catalog filter wrongly drops every FK whose real // parent is in another catalog — e.g. getImportedKeys (no parent supplied) returns empty while // Thrift returns the row. Parent schema/table are not seeded, so they already no-op when unset // and are passed through as-is; only the catalog needs this guard. @@ -1923,111 +1808,25 @@ private async Task GetCrossReferenceAsyncNoThrow( string sql = new ShowForeignKeysCommand(fkCatalog!, fkSchema!, fkTable!).Build(); activity?.SetTag("sql_query", sql); - List batches; - try - { - batches = await _connection.ExecuteMetadataSqlAsync(sql, cancellationToken).ConfigureAwait(false); - } - catch (DatabricksException ex) when (ex.IsObjectNotFoundException()) - { - return MetadataSchemaFactory.CreateEmptyCrossReferenceResult(); - } + var result = await _connection.ReadMetadataAsync( + sql, MetadataOperation.GetCrossReference, cancellationToken).ConfigureAwait(false); + IsNativeMetadataResult = result.IsNative; + if (result.IsNative) + return NativeMetadataResultBuilder.Build( + result, MetadataSchemaFactory.CreateCrossReferenceSchema(), MetadataOperation.GetCrossReference, + parentCatalog: pkCatalog, parentSchema: pkSchema, parentTable: pkTable); - var refs = new List<(string, string, string, string, string, string, string, string, int, int, int, string, string?, int)>(); - int seq = 0; - foreach (var batch in batches) - { - var pkCatalogArray = TryGetColumn(batch, "parentCatalogName"); - var pkSchemaArray = TryGetColumn(batch, "parentNamespace"); - var pkTableArray = TryGetColumn(batch, "parentTableName"); - var pkColArray = TryGetColumn(batch, "parentColName"); - var fkCatalogArray = TryGetColumn(batch, "catalogName"); - var fkSchemaArray = TryGetColumn(batch, "namespace"); - var fkTableArray = TryGetColumn(batch, "tableName"); - var fkColArray = TryGetColumn(batch, "col_name"); - var fkNameArray = TryGetColumn(batch, "constraintName"); - var fkKeySeqArray = TryGetColumn(batch, "keySeq"); - var fkUpdateRuleArray = TryGetColumn(batch, "updateRule"); - var fkDeleteRuleArray = TryGetColumn(batch, "deleteRule"); - var fkDeferrabilityArray = TryGetColumn(batch, "deferrability"); - - if (fkColArray == null) continue; - - for (int i = 0; i < batch.Length; i++) - { - if (fkColArray.IsNull(i)) continue; - - // Raw server-provided parent identifiers, null when the column is - // absent or NULL for this row. Filtering compares against these raw - // values (not the value-population fallback below) so a requested - // parent can never match a null server column against itself. - string? srvPkCatalog = pkCatalogArray != null && !pkCatalogArray.IsNull(i) ? pkCatalogArray.GetString(i) : null; - string? srvPkSchema = pkSchemaArray != null && !pkSchemaArray.IsNull(i) ? pkSchemaArray.GetString(i) : null; - string? srvPkTable = pkTableArray != null && !pkTableArray.IsNull(i) ? pkTableArray.GetString(i) : null; - - // SHOW FOREIGN KEYS is scoped to the FOREIGN table only, so it returns - // FKs to every parent. When the caller specified parent identifiers, - // filter the rows down to that parent — mirroring the JDBC reference - // driver (CrossReferenceKeysDatabricksResultSetAdapter.includeRow), which - // keeps a row only when its parent catalog/schema/table equalsIgnoreCase - // the requested one. A null parent arg means "no parent constraint" (the - // GetColumnsExtended foreign-only reuse passes null on all three), so it - // is NOT a filter; a specified parent that the server row doesn't match - // (including a null/empty server value) filters the row out. - if (!ParentMatches(pkCatalog, srvPkCatalog) - || !ParentMatches(pkSchema, srvPkSchema) - || !ParentMatches(pkTable, srvPkTable)) - { - continue; - } - - // Value population: fall back to the requested parent value when the - // server column is null so the emitted row still carries an identifier. - var rowPkCatalog = srvPkCatalog ?? pkCatalog ?? ""; - var rowPkSchema = srvPkSchema ?? pkSchema ?? ""; - var rowPkTable = srvPkTable ?? pkTable ?? ""; - - refs.Add(( - rowPkCatalog, - rowPkSchema, - rowPkTable, - pkColArray != null && !pkColArray.IsNull(i) ? pkColArray.GetString(i) : "", - fkCatalogArray != null && !fkCatalogArray.IsNull(i) ? fkCatalogArray.GetString(i) : fkCatalog!, - fkSchemaArray != null && !fkSchemaArray.IsNull(i) ? fkSchemaArray.GetString(i) : fkSchema!, - fkTableArray != null && !fkTableArray.IsNull(i) ? fkTableArray.GetString(i) : fkTable!, - fkColArray.GetString(i), - fkKeySeqArray != null && !fkKeySeqArray.IsNull(i) ? fkKeySeqArray.GetValue(i)!.Value : ++seq, - fkUpdateRuleArray != null && !fkUpdateRuleArray.IsNull(i) ? fkUpdateRuleArray.GetValue(i)!.Value : 0, - fkDeleteRuleArray != null && !fkDeleteRuleArray.IsNull(i) ? fkDeleteRuleArray.GetValue(i)!.Value : 0, - fkNameArray != null && !fkNameArray.IsNull(i) ? fkNameArray.GetString(i) : "", - (string?)null, - fkDeferrabilityArray != null && !fkDeferrabilityArray.IsNull(i) ? fkDeferrabilityArray.GetValue(i)!.Value : 5 - )); - } - } + var refs = MetadataRowReader.ForeignKeys(result, pkCatalog, pkSchema, pkTable, + fkCatalog!, fkSchema!, fkTable!) + .Select(row => (row.ParentCatalog, row.ParentSchema, row.ParentTable, row.ParentColumn, + row.Catalog, row.Schema, row.Table, row.Column, row.Sequence, row.UpdateRule, + row.DeleteRule, row.Name, row.ParentName, row.Deferrability)).ToList(); activity?.SetTag("result_count", refs.Count); return MetadataSchemaFactory.BuildCrossReferenceResult(refs); }, "GetCrossReference").ConfigureAwait(false); } - private static T? TryGetColumn(RecordBatch batch, string name) where T : class, IArrowArray - { - try { return batch.Column(name) as T; } - catch (ArgumentOutOfRangeException) { return null; } - } - - /// - /// Cross-reference parent-identifier filter. A null means - /// "no parent constraint" (matches any row); a specified value matches case-insensitively, - /// mirroring the JDBC reference driver's CrossReferenceKeysDatabricksResultSetAdapter. - /// An empty-string requested value therefore matches only an empty-string row value — - /// which no real FK has — so it correctly filters everything out. A null row value - /// (server column absent/NULL) never matches a specified parent. - /// - private static bool ParentMatches(string? requested, string? rowValue) - => requested == null || (rowValue != null && string.Equals(requested, rowValue, StringComparison.OrdinalIgnoreCase)); - // TracingStatement implementation public override string AssemblyVersion => GetType().Assembly.GetName().Version?.ToString() ?? "1.0.0"; public override string AssemblyName => "AdbcDrivers.Databricks"; diff --git a/csharp/test/ColumnMetadataHelperTests.cs b/csharp/test/ColumnMetadataHelperTests.cs index e4c3610c..55e509f1 100644 --- a/csharp/test/ColumnMetadataHelperTests.cs +++ b/csharp/test/ColumnMetadataHelperTests.cs @@ -15,6 +15,7 @@ */ using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2.Hive2; using Xunit; using static AdbcDrivers.HiveServer2.Hive2.HiveServer2Connection; @@ -202,5 +203,20 @@ public void GetDataTypeCode_HandlesTrimAndCase(string typeName, short expectedCo Assert.Equal(expectedCode, ColumnMetadataHelper.GetDataTypeCode(typeName)); } + [Theory] + [InlineData("DECIMAL(10,2)", 10, 2)] + [InlineData("DOUBLE", 8, 15)] + [InlineData("VARCHAR(42)", 42, 0)] + [InlineData("BINARY", 0, 0)] + public void PopulateTableInfo_PreservesComputedPrecisionAndScale(string typeName, int size, int digits) + { + var tableInfo = new TableInfo("TABLE"); + + ColumnMetadataHelper.PopulateTableInfoFromTypeName(tableInfo, "a", typeName, 0); + + Assert.Equal(size, Assert.Single(tableInfo.Precision)); + Assert.Equal((short)digits, Assert.Single(tableInfo.Scale)); + } + } } diff --git a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs index fe616441..474c50be 100644 --- a/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs +++ b/csharp/test/E2E/StatementExecution/SeaMetadataE2ETests.cs @@ -19,9 +19,11 @@ using System.IO; using System.Text.Json; using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; using Apache.Arrow; using Apache.Arrow.Adbc; using Apache.Arrow.Adbc.Tests; +using Apache.Arrow.Adbc.Tests.Metadata; using AdbcDrivers.HiveServer2; using Xunit; using Xunit.Abstractions; @@ -63,6 +65,25 @@ private void SkipIfNotConfigured() Skip.IfNot(Utils.CanExecuteTestConfig(TestConfigVariable), "Test configuration not available"); } + private async Task CreateMetadataRecordingConnection() + { + var parameters = GetDriverParameters(TestConfiguration); + parameters[DatabricksParameters.Protocol] = "rest"; + parameters[DatabricksParameters.EnableMultipleCatalogSupport] = "true"; + var connection = new MetadataRecordingConnection(parameters); + try + { + await connection.OpenAsync(); + connection.Statements.Clear(); + return connection; + } + catch + { + connection.Dispose(); + throw; + } + } + // Connection on whatever protocol the test suite was configured with (driver // parameters / config file). Tests never pick the protocol themselves — the run // argument decides it, so the SEA/Reyden nightly runs them over REST and the @@ -84,7 +105,8 @@ private AdbcConnection CreateConnection(Dictionary? extraParams private async Task>> ReadMetadata(AdbcConnection connection, string command, string? catalog = null, string? schema = null, string? table = null, string? column = null, - string? tableTypes = null, bool escapeWildcards = false) + string? tableTypes = null, bool escapeWildcards = false, bool requireNative = false, + string? foreignTable = null) { var results = new List>(); using var stmt = connection.CreateStatement(); @@ -95,10 +117,21 @@ private async Task>> ReadMetadata(AdbcConnection if (table != null) stmt.SetOption(ApacheParameters.TableName, table); if (column != null) stmt.SetOption(ApacheParameters.ColumnName, column); if (tableTypes != null) stmt.SetOption(ApacheParameters.TableTypes, tableTypes); + if (foreignTable != null) + { + stmt.SetOption(ApacheParameters.ForeignCatalogName, catalog!); + stmt.SetOption(ApacheParameters.ForeignSchemaName, schema!); + stmt.SetOption(ApacheParameters.ForeignTableName, foreignTable); + } stmt.SqlQuery = command; var result = stmt.ExecuteQuery(); using var reader = result.Stream; + if (requireNative) + { + Assert.True(Assert.IsType(stmt).IsNativeMetadataResult, + $"{command} fell back to SHOW; this warehouse must return is_native_metadata_result: true."); + } while (true) { @@ -200,6 +233,58 @@ public async Task GetTables_TypesFilter_IsCaseSensitive() private static readonly Dictionary RestProtocol = new() { { DatabricksParameters.Protocol, "rest" } }; + [SkippableTheory] + [InlineData("GetCatalogs", "all_column_types", null, "TABLE_CAT", "main")] + [InlineData("GetSchemas", "all_column_types", null, "TABLE_SCHEM", TestSchema)] + [InlineData("GetTables", "all_column_types", null, "TABLE_NAME", "all_column_types")] + [InlineData("GetColumns", "all_column_types", null, "COLUMN_NAME", "c_int")] + [InlineData("GetPrimaryKeys", "cross_ref_customers", null, "TABLE_NAME", "cross_ref_customers")] + [InlineData("GetCrossReference", "cross_ref_customers", "cross_ref_orders", "PKTABLE_NAME", "cross_ref_customers")] + public async Task NativeMetadata_UsesNativeResponse( + string command, string table, string? foreignTable, string expectedColumn, string expectedValue) + { + SkipIfNotConfigured(); + + using var conn = CreateConnection(new Dictionary(RestProtocol) + { + [DatabricksParameters.EnableMultipleCatalogSupport] = "true", + [DatabricksParameters.EnablePKFK] = "true", + }); + var rows = await ReadMetadata(conn, command, TestCatalog, TestSchema, table, + requireNative: true, foreignTable: foreignTable); + + Assert.Contains(rows, row => row[expectedColumn] == expectedValue); + } + + [SkippableFact] + public async Task NativeMetadata_GetObjects_UsesNativeResponse() + { + SkipIfNotConfigured(); + using var connection = await CreateMetadataRecordingConnection(); + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, TestCatalog, TestSchema, TestTable, null, null); + using var batch = await stream.ReadNextRecordBatchAsync(); + Assert.NotNull(batch); + + var catalog = Assert.Single(GetObjectsParser.ParseCatalog(batch, null), + catalog => catalog.Name == TestCatalog); + var schema = Assert.Single(catalog.DbSchemas!, schema => schema.Name == TestSchema); + var table = Assert.Single(schema.Tables!, table => table.Name == TestTable); + Assert.Contains(table.Columns!, column => column.Name == "c_int"); + connection.AssertNativeResponses("SHOW CATALOGS", "SHOW SCHEMAS", "SHOW TABLES", "SHOW COLUMNS"); + } + + [SkippableFact] + public async Task NativeMetadata_GetTableSchema_UsesNativeResponse() + { + SkipIfNotConfigured(); + using var connection = await CreateMetadataRecordingConnection(); + var schema = connection.GetTableSchema(TestCatalog, TestSchema, "cross_ref_customers"); + + Assert.Equal("customer_id", schema.FieldsList[0].Name); + connection.AssertNativeResponses("SHOW COLUMNS"); + } + [SkippableFact] public async Task GetColumns_CatalogMatchAll_WithEscaping_ReturnsEmpty() { diff --git a/csharp/test/MetadataRecordingConnection.cs b/csharp/test/MetadataRecordingConnection.cs new file mode 100644 index 00000000..9f023366 --- /dev/null +++ b/csharp/test/MetadataRecordingConnection.cs @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Net.Http; +using AdbcDrivers.Databricks.StatementExecution; +using Apache.Arrow.Adbc; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests +{ + internal sealed class MetadataRecordingConnection : StatementExecutionConnection + { + internal MetadataRecordingConnection(IReadOnlyDictionary properties) + : base(properties) + { + } + + internal MetadataRecordingConnection(IReadOnlyDictionary properties, HttpClient http) + : base(properties, http) + { + } + + internal List Statements { get; } = new(); + + public override AdbcStatement CreateStatement() + { + var statement = (StatementExecutionStatement)base.CreateStatement(); + Statements.Add(statement); + return statement; + } + + internal void AssertNativeResponses(params string[] commands) + { + Assert.NotEmpty(Statements); + foreach (string command in commands) + Assert.Contains(Statements, statement => + statement.SqlQuery?.StartsWith(command, StringComparison.Ordinal) == true); + Assert.All(Statements, statement => Assert.True(statement.IsNativeMetadataResult, + $"{statement.SqlQuery} fell back to SHOW; expected is_native_metadata_result: true.")); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs new file mode 100644 index 00000000..63e751ae --- /dev/null +++ b/csharp/test/Unit/StatementExecution/MetadataRowReaderTests.cs @@ -0,0 +1,385 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Types; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class MetadataRowReaderTests + { + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Catalogs_OnlyFilterNativeResponses(bool native) + { + using var batch = Strings((native ? "TABLE_CAT" : "catalog", + new string?[] { "ma_catalog", "maxcatalog", "other" })); + var result = new MetadataBatches(new List { batch }, native, + catalogFilter: CatalogFilter.Pattern(@"ma\_%")); + + string[] expected = native + ? new[] { "ma_catalog" } + : new[] { "ma_catalog", "maxcatalog", "other" }; + Assert.Equal(expected, MetadataRowReader.Catalogs(result)); + } + + [Theory] + [InlineData(false, "")] + [InlineData(true, "TABLE")] + public void ShowTables_EmptyTypeKeepsExistingCallSiteBehavior(bool normalizeEmptyTableType, string expectedType) + { + using var batch = Strings( + ("catalogName", new string?[] { "main" }), + ("namespace", new string?[] { "default" }), + ("tableName", new string?[] { "t" }), + ("tableType", new string?[] { "" })); + var result = new MetadataBatches(new List { batch }, false); + + var row = Assert.Single(MetadataRowReader.Tables( + result, normalizeEmptyTableType: normalizeEmptyTableType)); + Assert.Equal(expectedType, row.TableType); + var filtered = MetadataRowReader.Tables( + result, new[] { "TABLE" }, normalizeEmptyTableType: normalizeEmptyTableType); + Assert.Equal(normalizeEmptyTableType ? 1 : 0, filtered.Count); + } + + [Fact] + public void ShowColumns_OrdinalsContinueAcrossBatchesPerTable() + { + using var first = ShowColumns(("t", "a"), ("other", "b"), ("t", "c")); + using var second = ShowColumns(("t", "d")); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { first, second }, false), + }); + + Assert.Equal(new[] { 0, 0, 1, 2 }, result.Rows.Select(row => row.Ordinal)); + Assert.All(result.Rows, row => Assert.True(row.Nullable)); + Assert.False(result.IsNative); + } + + [Fact] + public void ShowColumns_PartialIdentifiersStillProvideTableSchemaFields() + { + using var batch = Strings( + ("col_name", new string?[] { "a" }), + ("columnType", new string?[] { "INT" }), + ("isNullable", new string?[] { "false" })); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { batch }, false), + }); + + var row = Assert.Single(result.Rows); + Assert.Null(row.Catalog); + Assert.Null(row.Schema); + Assert.Null(row.Table); + Assert.Equal("a", row.Name); + Assert.Equal("INT", row.TypeName); + Assert.False(row.Nullable); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NativeColumns_ShareCatalogFilterAndConcreteFallback(bool pattern) + { + using var batch = NativeColumns("foo_bar", "fooxbar", null); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { batch }, true, "foo_bar", + pattern ? CatalogFilter.Pattern("foo_bar") : CatalogFilter.Exact("foo_bar")), + }); + + Assert.True(result.IsNative); + Assert.Equal(3, Assert.Single(Assert.Single(result.Results).Batches).Length); + Assert.Equal(pattern ? new[] { "foo_bar", "fooxbar" } : new[] { "foo_bar", "foo_bar" }, + result.Rows.Select(row => row.Catalog)); + Assert.All(result.Rows, row => + { + Assert.Equal(7, row.Ordinal); + Assert.Equal("7", row.Default); + Assert.True(row.Nullable); + Assert.True(row.IsAutoIncrement); + }); + + var flat = NativeMetadataResultBuilder.Build(result.Results, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(result.Rows.Count, output.Length); + var catalogs = (StringArray)output.Column("TABLE_CAT"); + Assert.Equal(result.Rows.Select(row => row.Catalog), + Enumerable.Range(0, output.Length).Select(row => catalogs.GetString(row))); + } + + public static IEnumerable NativeCatalogFallbacks() + { + foreach (var operation in new[] + { + MetadataOperation.GetSchemas, MetadataOperation.GetTables, MetadataOperation.GetColumns, + }) + foreach (var (pattern, scope, expected) in new[] + { + (false, "foo_bar", "foo_bar"), + (true, "main", "main"), + (true, @"foo\_bar", "foo_bar"), + (true, @"foo\%bar", "foo%bar"), + (true, "ma%", (string?)null), + (true, "foo_bar", (string?)null), + }) + yield return new object?[] { operation.ToString(), pattern, scope, expected }; + } + + [Theory] + [MemberData(nameof(NativeCatalogFallbacks))] + public async Task MissingNativeCatalog_OnlyUsesConcreteScope( + string operationName, bool pattern, string scope, string? expectedCatalog) + { + var operation = (MetadataOperation)System.Enum.Parse(typeof(MetadataOperation), operationName); + using var batch = operation switch + { + MetadataOperation.GetSchemas => Strings( + ("TABLE_SCHEM", new string?[] { "default" }), + ("TABLE_CATALOG", new string?[] { null })), + MetadataOperation.GetTables => Strings(MetadataSchemaFactory.CreateTablesSchema().FieldsList + .Select(field => (field.Name, new string?[] + { + field.Name switch + { + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "TABLE_TYPE" => "TABLE", + _ => null, + }, + })).ToArray()), + _ => NativeColumns(new string?[] { null }), + }; + var result = new MetadataBatches(new List { batch }, true, scope, + pattern ? CatalogFilter.Pattern(scope) : CatalogFilter.Exact(scope)); + var catalogs = operation switch + { + MetadataOperation.GetSchemas => MetadataRowReader.Schemas(result).Select(row => row.Catalog), + MetadataOperation.GetTables => MetadataRowReader.Tables(result).Select(row => row.Catalog), + _ => MetadataRowReader.Columns(new[] { result }).Select(row => row.Catalog), + }; + var expected = expectedCatalog == null ? System.Array.Empty() : new[] { expectedCatalog }; + Assert.Equal(expected, catalogs); + + var flat = NativeMetadataResultBuilder.Build(result, batch.Schema, operation); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(expected.Length, output.Length); + if (expectedCatalog != null) + Assert.Equal(expectedCatalog, ((StringArray)output.Column( + operation == MetadataOperation.GetSchemas ? "TABLE_CATALOG" : "TABLE_CAT")).GetString(0)); + } + + [Fact] + public async Task MixedColumns_PreserveServerOrdinalsAndCountShowRows() + { + using var native = NativeColumns("main"); + using var show = ShowColumns(("t", "b")); + var result = new ColumnMetadataResult(new[] + { + new MetadataBatches(new List { native }, true, "main"), + new MetadataBatches(new List { show }, false, "main"), + }); + + Assert.False(result.IsNative); + Assert.Equal(new[] { 7, 1 }, result.Rows.Select(row => row.Ordinal)); + Assert.Equal(new[] { "a", "b" }, result.Rows.Select(row => row.Name)); + + var flat = await FlatColumnsResultBuilder.BuildFlatColumnsResultAsync( + result, System.Threading.CancellationToken.None); + using var stream = flat.Stream!; + using var output = (await stream.ReadNextRecordBatchAsync())!; + var positions = (Int32Array)output.Column("ORDINAL_POSITION"); + Assert.Equal(new[] { 7, 1 }, Enumerable.Range(0, output.Length).Select(row => positions.GetValue(row)!.Value)); + } + + [Fact] + public void ShowPrimaryKeys_PreserveCanonicalNamesAndSequenceFallbackAcrossBatches() + { + using var firstStrings = Strings( + ("col_name", new string?[] { "a", "b", null }), + ("catalogName", new string?[] { "Main", null, null }), + ("namespace", new string?[] { "Default", null, null }), + ("tableName", new string?[] { "Table", null, null }), + ("constraintName", new string?[] { "pk", null, null })); + using var sequences = new Int32Array.Builder().Append(7).AppendNull().AppendNull().Build(); + using var first = new RecordBatch( + new Schema(firstStrings.Schema.FieldsList.Concat(new[] { new Field("keySeq", Int32Type.Default, true) }), null), + Enumerable.Range(0, firstStrings.ColumnCount).Select(firstStrings.Column).Concat(new[] { sequences }).ToArray(), 3); + using var second = Strings(("col_name", new string?[] { "c" })); + var result = new MetadataBatches(new List { first, second }, false); + + var rows = MetadataRowReader.PrimaryKeys(result, "main", "default", "table").ToList(); + + Assert.Equal(new[] { 7, 1, 2 }, rows.Select(row => row.Sequence)); + Assert.Equal(new[] { "Main", "main", "main" }, rows.Select(row => row.Catalog)); + Assert.Equal(new[] { "Default", "default", "default" }, rows.Select(row => row.Schema)); + Assert.Equal(new[] { "Table", "table", "table" }, rows.Select(row => row.Table)); + Assert.Equal(new[] { "pk", "", "" }, rows.Select(row => row.Name)); + } + + [Fact] + public void ShowForeignKeys_FilterRawParentBeforeDefaults() + { + using var batch = Strings( + ("parentCatalogName", new string?[] { "Main", null }), + ("parentNamespace", new string?[] { "Sales", null }), + ("parentTableName", new string?[] { "Customers", null }), + ("col_name", new string?[] { "customer_id", "unknown" })); + var result = new MetadataBatches(new List { batch }, false); + + var row = Assert.Single(MetadataRowReader.ForeignKeys( + result, "main", "sales", "customers", "foreign", "default", "orders")); + + Assert.Equal("Main", row.ParentCatalog); + Assert.Equal("Sales", row.ParentSchema); + Assert.Equal("Customers", row.ParentTable); + Assert.Equal("foreign", row.Catalog); + Assert.Equal("default", row.Schema); + Assert.Equal("orders", row.Table); + Assert.Equal(1, row.Sequence); + Assert.Equal(0, row.UpdateRule); + Assert.Equal(0, row.DeleteRule); + Assert.Equal(5, row.Deferrability); + Assert.Equal("", row.Name); + Assert.Null(row.ParentName); + Assert.Equal(2, MetadataRowReader.ForeignKeys( + result, null, null, null, "foreign", "default", "orders").Count()); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task NativeKeys_DecodedAndFlatRowsPreserveServerFields(bool foreign) + { + var operation = foreign ? MetadataOperation.GetCrossReference : MetadataOperation.GetPrimaryKeys; + var schema = foreign ? MetadataSchemaFactory.CreateCrossReferenceSchema() : MetadataSchemaFactory.CreatePrimaryKeysSchema(); + schema = new Schema(schema.FieldsList.Reverse(), null); + var arrays = schema.FieldsList.Select(field => field.DataType.TypeId == ArrowTypeId.String + ? (IArrowArray)new StringArray.Builder().Append(field.Name switch + { + "TABLE_CAT" or "PKTABLE_CAT" => "Main", + "TABLE_SCHEM" or "PKTABLE_SCHEM" => "Sales", + "TABLE_NAME" or "PKTABLE_NAME" => "Customers", + _ => field.Name, + }).Build() + : new Int32Array.Builder().Append(7).Build()).ToArray(); + using var batch = new RecordBatch(schema, arrays, 1); + var response = new MetadataBatches(new List { batch }, true); + + var result = NativeMetadataResultBuilder.Build(response, schema, operation, + parentCatalog: "main", parentSchema: "sales", parentTable: "customers"); + using var stream = result.Stream!; + using var flat = (await stream.ReadNextRecordBatchAsync())!; + Assert.Equal(7, ((Int32Array)flat.Column("KEQ_SEQ")).GetValue(0)); + Assert.Equal("PK_NAME", ((StringArray)flat.Column("PK_NAME")).GetString(0)); + + if (foreign) + { + var row = Assert.Single(MetadataRowReader.ForeignKeys( + response, "main", "sales", "customers", "foreign", "default", "orders")); + Assert.Equal("Main", row.ParentCatalog); + Assert.Equal("PK_NAME", row.ParentName); + Assert.Equal("FKTABLE_CAT", row.Catalog); + Assert.Equal(7, row.Sequence); + Assert.Empty(MetadataRowReader.ForeignKeys( + response, "other", "sales", "customers", "foreign", "default", "orders")); + var filtered = NativeMetadataResultBuilder.Build(response, schema, operation, parentCatalog: "other"); + using var filteredStream = filtered.Stream!; + Assert.Equal(0, filtered.RowCount); + } + else + { + var row = Assert.Single(MetadataRowReader.PrimaryKeys(response, "fallback", "fallback", "fallback")); + Assert.Equal("Main", row.Catalog); + Assert.Equal("Sales", row.Schema); + Assert.Equal("Customers", row.Table); + Assert.Equal("PK_NAME", row.Name); + Assert.Equal(7, row.Sequence); + } + } + + private static RecordBatch ShowColumns(params (string Table, string Name)[] rows) + => Strings( + ("catalogName", rows.Select(_ => (string?)"main").ToArray()), + ("namespace", rows.Select(_ => (string?)"default").ToArray()), + ("tableName", rows.Select(row => (string?)row.Table).ToArray()), + ("col_name", rows.Select(row => (string?)row.Name).ToArray()), + ("columnType", rows.Select(_ => (string?)"INT").ToArray())); + + private static RecordBatch Strings(params (string Name, string?[] Values)[] columns) + { + var schema = new Schema(columns.Select(column => + new Field(column.Name, StringType.Default, true)), null); + var arrays = columns.Select(column => + { + var builder = new StringArray.Builder(); + foreach (string? value in column.Values) + { + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + }).ToArray(); + return new RecordBatch(schema, arrays, columns[0].Values.Length); + } + + private static RecordBatch NativeColumns(params string?[] catalogs) + { + var schema = new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23), null); + var arrays = schema.FieldsList.Select(field => + { + if (field.DataType.TypeId == ArrowTypeId.String) + { + var builder = new StringArray.Builder(); + foreach (string? catalog in catalogs) + { + string? value = field.Name switch + { + "TABLE_CAT" => catalog, + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "COLUMN_NAME" => "a", + "TYPE_NAME" => "INT", + "COLUMN_DEF" => "7", + "IS_AUTO_INCREMENT" => "YES", + _ => null, + }; + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + } + int number = field.Name == "ORDINAL_POSITION" ? 7 : field.Name == "NULLABLE" ? 1 : 0; + return field.DataType.TypeId == ArrowTypeId.Int16 + ? (IArrowArray)new Int16Array.Builder().AppendRange( + Enumerable.Repeat((short)number, catalogs.Length)).Build() + : new Int32Array.Builder().AppendRange(Enumerable.Repeat(number, catalogs.Length)).Build(); + }).ToArray(); + return new RecordBatch(schema, arrays, catalogs.Length); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs new file mode 100644 index 00000000..7fa8c3c6 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/NativeMetadataResultBuilderTests.cs @@ -0,0 +1,291 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using Apache.Arrow; +using Apache.Arrow.Types; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class NativeMetadataResultBuilderTests + { + [Theory] + [InlineData("DECIMAL(10,2)", 0, 0, 10, 2)] + [InlineData("INT", 10, 0, 10, 0)] + [InlineData("FLOAT", 7, 7, 7, 7)] + [InlineData("BINARY", 99, 3, 99, 3)] + [InlineData("VARCHAR(42)", 99, 3, 42, 3)] + [InlineData("CHAR(4)", 99, 3, 4, 3)] + [InlineData("DOUBLE", null, null, 0, 0)] + [InlineData("DOUBLE", 15, null, 15, 0)] + [InlineData("DOUBLE", null, 15, 0, 15)] + [InlineData("DECIMAL(10,2)", null, null, 10, 2)] + [InlineData("VARCHAR(42)", null, null, 42, 0)] + public async Task GetColumns_OnlyNormalizesThriftPrecisionAndScale( + string typeName, int? columnSize, int? decimalDigits, int expectedSize, int expectedScale) + { + var target = new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Reverse(), null); + Assert.Equal(24, target.FieldsList.Count); + var fields = target.FieldsList.Where(field => field.Name != "BASE_TYPE_NAME").Select(field => + field.Name == "ORDINAL_POSITION" + ? new Field(field.Name, Int64Type.Default, true) + : field).ToArray(); + var source = new Schema(fields, null); + var arrays = new List(); + foreach (var field in fields) + { + if (field.Name == "ORDINAL_POSITION") + arrays.Add(new Int64Array.Builder().Append(0).Build()); + else if (field.Name == "TABLE_CAT") + arrays.Add(new StringArray.Builder().AppendNull().Build()); + else if (field.DataType.TypeId == ArrowTypeId.String) + arrays.Add(new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? typeName : "value").Build()); + else if (field.DataType.TypeId == ArrowTypeId.Int8) + arrays.Add(new Int8Array.Builder().Append(0).Build()); + else if (field.DataType.TypeId == ArrowTypeId.Int16) + arrays.Add(new Int16Array.Builder().Append(0).Build()); + else if (field.Name == "BUFFER_LENGTH") + arrays.Add(new Int32Array.Builder().AppendNull().Build()); + else + arrays.Add(new Int32Array.Builder().Append(field.Name == "COLUMN_SIZE" ? columnSize + : field.Name == "DECIMAL_DIGITS" ? decimalDigits : 0).Build()); + } + + using var nativeBatch = new RecordBatch(source, arrays.ToArray(), 1); + var result = NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true, "main"), + target, MetadataOperation.GetColumns); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + Assert.Equal(expectedSize, ((Int32Array)batch.Column("COLUMN_SIZE")).GetValue(0)); + Assert.True(batch.Column("BUFFER_LENGTH").IsNull(0)); + Assert.Equal(expectedScale, ((Int32Array)batch.Column("DECIMAL_DIGITS")).GetValue(0)); + Assert.Equal(0, ((Int32Array)batch.Column("ORDINAL_POSITION")).GetValue(0)); + Assert.Equal("main", ((StringArray)batch.Column("TABLE_CAT")).GetString(0)); + Assert.Equal(ColumnMetadataHelper.GetBaseTypeName(typeName), ((StringArray)batch.Column("BASE_TYPE_NAME")).GetString(0)); + } + + [Fact] + public async Task GetTables_DefaultsEmptyTypeBeforeFilteringAndFillsRequestedCatalog() + { + var schema = new Schema(MetadataSchemaFactory.CreateTablesSchema().FieldsList.Reverse(), null); + var arrays = schema.FieldsList.Select(field => (IArrowArray)(field.Name switch + { + "TABLE_CAT" => new StringArray.Builder().AppendNull().Build(), + "TABLE_TYPE" => new StringArray.Builder().Append("").Build(), + _ => new StringArray.Builder().Append(field.Name).Build(), + })).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + var result = NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true, "main", CatalogFilter.Exact("main")), + schema, MetadataOperation.GetTables, tableTypes: new[] { "TABLE" }); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.Equal(1, result.RowCount); + Assert.Equal("main", ((StringArray)batch!.Column("TABLE_CAT")).GetString(0)); + Assert.Equal("TABLE", ((StringArray)batch.Column("TABLE_TYPE")).GetString(0)); + } + + [Fact] + public async Task GetSchemas_FillsMissingCatalogByFieldName() + { + var schema = new Schema(MetadataSchemaFactory.CreateSchemasSchema().FieldsList.Reverse(), null); + using var nativeBatch = new RecordBatch(schema, new IArrowArray[] + { + new StringArray.Builder().AppendNull().Build(), + new StringArray.Builder().Append("default").Build(), + }, 1); + var response = new MetadataBatches(new List { nativeBatch }, true, + "main", CatalogFilter.Exact("main")); + + var result = NativeMetadataResultBuilder.Build(response, schema, MetadataOperation.GetSchemas); + using var reader = result.Stream!; + using var batch = (await reader.ReadNextRecordBatchAsync())!; + + Assert.Equal("main", ((StringArray)batch.Column("TABLE_CATALOG")).GetString(0)); + Assert.Equal("default", ((StringArray)batch.Column("TABLE_SCHEM")).GetString(0)); + var decoded = Assert.Single(MetadataRowReader.Schemas(response)); + Assert.Equal("main", decoded.Catalog); + Assert.Equal("default", decoded.Schema); + } + + [Theory] + [InlineData(null, false)] + [InlineData(null, true)] + [InlineData("main", false)] + [InlineData("main", true)] + public async Task GetTables_FlatAndDecodedResultsShareCatalogFilteringAndOrdering( + string? catalog, bool filterTypes) + { + var schema = MetadataSchemaFactory.CreateTablesSchema(); + (string? Catalog, string Schema, string Table, string? Type)[] tables = + { + ("other", "a", "other_table", "TABLE"), + (null, "b", "b_a", ""), + ("main", "a", "a_view", "VIEW"), + ("main", "a", "a_z", "TABLE"), + ("main", "a", "a_a", null), + ("main", "b", "b_z", "TABLE"), + ("main", "a", "a_b", "TABLE"), + }; + var arrays = schema.FieldsList.Select(field => + { + var builder = new StringArray.Builder(); + foreach (var table in tables) + { + string? value = field.Name switch + { + "TABLE_CAT" => table.Catalog, + "TABLE_SCHEM" => table.Schema, + "TABLE_NAME" => table.Table, + "TABLE_TYPE" => table.Type, + _ => "server_value", + }; + if (value == null) builder.AppendNull(); else builder.Append(value); + } + return (IArrowArray)builder.Build(); + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, tables.Length); + string[]? tableTypes = filterTypes ? new[] { "TABLE" } : null; + var response = new MetadataBatches(new List { nativeBatch }, true, + catalog, CatalogFilter.Exact(catalog)); + var decoded = MetadataRowReader.Tables(response, tableTypes); + var result = NativeMetadataResultBuilder.Build( + response, schema, MetadataOperation.GetTables, tableTypes: tableTypes); + using var reader = result.Stream!; + using var batch = (await reader.ReadNextRecordBatchAsync())!; + var names = (StringArray)batch.Column("TABLE_NAME"); + string[] expected = catalog == null + ? new[] { "b_a", "a_a", "a_b", "a_z", "b_z", "other_table" } + : new[] { "a_a", "a_b", "a_z", "b_a", "b_z" }; + if (!filterTypes) expected = expected.Concat(new[] { "a_view" }).ToArray(); + + Assert.Equal(expected, decoded.Select(row => row.Table)); + Assert.Equal(expected, Enumerable.Range(0, batch.Length).Select(row => names.GetString(row))); + Assert.All(Enumerable.Range(0, batch.Length), row => + Assert.Equal("server_value", ((StringArray)batch.Column("TYPE_CAT")).GetString(row))); + } + + [Theory] + [InlineData((int)MetadataOperation.GetSchemas, "%", true, 0)] + [InlineData((int)MetadataOperation.GetColumns, "compar%", true, 0)] + [InlineData((int)MetadataOperation.GetColumns, @"comparator\_tests", true, 0)] + [InlineData((int)MetadataOperation.GetSchemas, "comparator_tests", true, 1)] + [InlineData((int)MetadataOperation.GetColumns, "COMPARATOR_TESTS", true, 1)] + [InlineData((int)MetadataOperation.GetSchemas, "compar%", false, 2)] + public void NativeSchemasAndColumns_FilterLiteralCatalogsOnlyWhenRequested( + int operationCode, string requestedCatalog, bool requireExactCatalog, int expectedRows) + { + var operation = (MetadataOperation)operationCode; + var target = operation == MetadataOperation.GetSchemas + ? MetadataSchemaFactory.CreateSchemasSchema() + : MetadataSchemaFactory.CreateColumnMetadataSchema(); + var fields = operation == MetadataOperation.GetColumns + ? target.FieldsList.Take(23).ToArray() + : target.FieldsList.ToArray(); + string catalogField = operation == MetadataOperation.GetSchemas ? "TABLE_CATALOG" : "TABLE_CAT"; + var source = new Schema(fields, null); + var arrays = fields.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String when field.Name == catalogField => + (IArrowArray)new StringArray.Builder().Append("comparator_tests").Append("comparator-tests").Build(), + ArrowTypeId.String => new StringArray.Builder().Append(field.Name == "TYPE_NAME" ? "INT" : "value") + .Append(field.Name == "TYPE_NAME" ? "INT" : "value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Append(0).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(source, arrays, 2); + + var result = NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true, requestedCatalog, + CatalogFilter.Exact(requireExactCatalog ? requestedCatalog : null)), + target, operation); + + Assert.Equal(expectedRows, result.RowCount); + } + + [Theory] + [InlineData((int)MetadataOperation.GetCatalogs)] + [InlineData((int)MetadataOperation.GetSchemas)] + [InlineData((int)MetadataOperation.GetPrimaryKeys)] + [InlineData((int)MetadataOperation.GetCrossReference)] + public async Task OtherNativeOperations_PreserveTheirSchemas(int operationCode) + { + var operation = (MetadataOperation)operationCode; + var schema = operation switch + { + MetadataOperation.GetCatalogs => MetadataSchemaFactory.CreateCatalogsSchema(), + MetadataOperation.GetSchemas => MetadataSchemaFactory.CreateSchemasSchema(), + MetadataOperation.GetPrimaryKeys => MetadataSchemaFactory.CreatePrimaryKeysSchema(), + _ => MetadataSchemaFactory.CreateCrossReferenceSchema(), + }; + var arrays = schema.FieldsList.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder().Append("value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(1).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(1).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(1).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(1).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + var result = NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true), schema, operation); + using var reader = result.Stream!; + using var batch = await reader.ReadNextRecordBatchAsync(); + + Assert.Equal(1, result.RowCount); + Assert.Equal(schema.FieldsList.Count, batch!.ColumnCount); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GetColumns_RejectsWrongColumnNameOrType(bool wrongType) + { + var target = MetadataSchemaFactory.CreateColumnMetadataSchema(); + var fields = target.FieldsList.Take(23).Select(field => + field.Name == "COLUMN_NAME" + ? new Field(wrongType ? field.Name : "wrong_name", wrongType ? (IArrowType)Int32Type.Default : field.DataType, true) + : field).ToArray(); + var schema = new Schema(fields, null); + var arrays = fields.Select(field => field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder().Append("value").Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Build(), + }).ToArray(); + using var nativeBatch = new RecordBatch(schema, arrays, 1); + + Assert.Throws(() => NativeMetadataResultBuilder.Build( + new MetadataBatches(new List { nativeBatch }, true), target, MetadataOperation.GetColumns)); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs index dfe58cea..6366791a 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionClientTests.cs @@ -238,6 +238,59 @@ public async Task DeleteSessionAsync_WithHttpError_ThrowsDatabricksException() #region ExecuteStatementAsync Tests + [Fact] + public async Task ExecuteStatementAsync_NativeMetadata_SendsHeadersAndReadsManifest() + { + var request = new ExecuteStatementRequest + { + Statement = "SHOW TABLES IN CATALOG `main`", + WarehouseId = "warehouse-123", + IsMetadata = true, + MetadataOperation = MetadataOperation.GetTables, + }; + var responseJson = JsonSerializer.Serialize(new + { + statement_id = "stmt-native", + status = new { state = "SUCCEEDED" }, + manifest = new { is_native_metadata_result = true }, + }); + HttpRequestMessage? capturedRequest = null; + string? capturedContent = null; + SetupMockResponseWithCapture(HttpStatusCode.OK, responseJson, + req => capturedRequest = req, content => capturedContent = content); + + var response = await new StatementExecutionClient(_httpClient, _testHost) + .ExecuteStatementAsync(request, CancellationToken.None); + + Assert.Equal(true, response.Manifest?.IsNativeMetadataResult); + Assert.Equal("true", Assert.Single(capturedRequest!.Headers.GetValues("x-databricks-sea-can-run-fully-sync"))); + Assert.Equal("GetTables", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-metadata-operation-type"))); + Assert.Equal("true", Assert.Single(capturedRequest.Headers.GetValues("x-databricks-require-thrift-native-metadata"))); + using var requestJson = JsonDocument.Parse(capturedContent!); + foreach (var property in requestJson.RootElement.EnumerateObject()) + { + Assert.False(string.Equals(property.Name, "MetadataOperation", StringComparison.OrdinalIgnoreCase)); + Assert.False(string.Equals(property.Name, "IsNativeMetadataResult", StringComparison.OrdinalIgnoreCase)); + } + } + + [Fact] + public async Task ExecuteStatementAsync_MetadataWithoutOperation_OmitsNativeHeaders() + { + var request = new ExecuteStatementRequest { Statement = "DESC TABLE t", IsMetadata = true }; + HttpRequestMessage? capturedRequest = null; + SetupMockResponseWithCapture(HttpStatusCode.OK, + "{\"statement_id\":\"stmt-1\",\"status\":{\"state\":\"SUCCEEDED\"}}", + req => capturedRequest = req, _ => { }); + + await new StatementExecutionClient(_httpClient, _testHost) + .ExecuteStatementAsync(request, CancellationToken.None); + + Assert.True(capturedRequest!.Headers.Contains("x-databricks-sea-can-run-fully-sync")); + Assert.False(capturedRequest.Headers.Contains("x-databricks-metadata-operation-type")); + Assert.False(capturedRequest.Headers.Contains("x-databricks-require-thrift-native-metadata")); + } + [Fact] public async Task ExecuteStatementAsync_WithValidRequest_ReturnsResponse() { diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs index e94e98f3..ad0c781e 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionEmptyTableTypesTests.cs @@ -113,6 +113,74 @@ private static HttpClient HttpClientReturningShowTables() return new HttpClient(handler.Object); } + private static HttpClient HttpClientReturningNativeTables(List operations) + { + static StringArray ThreeNulls() + { + var builder = new StringArray.Builder(); + builder.AppendNull(); + builder.AppendNull(); + builder.AppendNull(); + return builder.Build(); + } + + string[] names = + { + "TABLE_CAT", "TABLE_SCHEM", "TABLE_NAME", "TABLE_TYPE", "REMARKS", + "TYPE_CAT", "TYPE_SCHEM", "TYPE_NAME", "SELF_REFERENCING_COL_NAME", "REF_GENERATION", + }; + var schema = new Schema(names.Select(name => new Field(name, StringType.Default, true)), null); + var arrays = new IArrowArray[] + { + new StringArray.Builder().Append("main").Append("main").Append("other").Build(), + new StringArray.Builder().Append("default").Append("default").Append("default").Build(), + new StringArray.Builder().Append("my_table").Append("my_view").Append("other_view").Build(), + new StringArray.Builder().Append("TABLE").Append("VIEW").Append("VIEW").Build(), + new StringArray.Builder().Append("table remark").Append("view remark").Append("other remark").Build(), + ThreeNulls(), ThreeNulls(), ThreeNulls(), ThreeNulls(), ThreeNulls(), + }; + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, arrays, 3)); + writer.WriteEnd(); + } + var executeBody = JsonSerializer.Serialize(new + { + statement_id = "stmt-native-tables", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = true, + total_row_count = 3, + schema = new + { + column_count = names.Length, + columns = names.Select((name, position) => new + { + name, position, type_name = "STRING", type_text = "STRING", + }).ToArray(), + }, + }, + result = new { attachment = raw.ToArray() }, + }); + var sessionBody = JsonSerializer.Serialize(new { session_id = "session-1" }); + var handler = new Mock(); + handler.Protected() + .Setup>( + "SendAsync", ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage req, CancellationToken _) => + { + if (req.RequestUri?.AbsolutePath.EndsWith("/api/2.0/sql/sessions") == true) + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(sessionBody) }; + if (req.Method != HttpMethod.Post) + return new HttpResponseMessage(HttpStatusCode.OK); + operations.Add(req.Headers.GetValues("x-databricks-metadata-operation-type").Single()); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(executeBody) }; + }); + return new HttpClient(handler.Object); + } + private static StatementExecutionConnection CreateConnection(HttpClient http) { var properties = new Dictionary @@ -173,6 +241,21 @@ public async Task GetTables_SpecificTableType_ReturnsOnlyThatType() Assert.Equal("TABLE", rows[0].tableType); } + [Fact] + public async Task GetTables_NativeResult_UsesThriftColumns() + { + var operations = new List(); + using var http = HttpClientReturningNativeTables(operations); + using var connection = CreateConnection(http); + + var rows = await ListTablesAsync(connection, new[] { "VIEW" }); + + Assert.Single(rows); + Assert.Equal("my_view", rows[0].table); + Assert.Equal("VIEW", rows[0].tableType); + Assert.Equal(new[] { "GetTables" }, operations); + } + // ─── is_metadata_command path (StatementExecutionStatement.GetTablesAsync) ────── // The metadata-command shim receives tableTypes as a pre-joined string option // (adbc.get_metadata.target_table_types). An empty string means "match none" @@ -231,5 +314,17 @@ public async Task MetadataCommand_SpecificTableType_ReturnsOnlyThatType() Assert.Single(types); Assert.Equal("TABLE", types[0]); } + + [Fact] + public async Task MetadataCommand_NativeTables_FiltersByType() + { + var operations = new List(); + using var http = HttpClientReturningNativeTables(operations); + + var types = await MetadataCommandTableTypes(http, "TABLE"); + + Assert.Equal(new[] { "TABLE" }, types); + Assert.Equal(new[] { "GetTables" }, operations); + } } } diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs index 6d15e4ce..3a9a1d6e 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataObjectNotFoundTests.cs @@ -15,6 +15,7 @@ */ using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Net; @@ -28,6 +29,7 @@ using Apache.Arrow.Types; using AdbcDrivers.Databricks.StatementExecution; using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; using AdbcDrivers.HiveServer2.Spark; using Microsoft.IO; using Moq; @@ -734,26 +736,22 @@ await Assert.ThrowsAsync( () => stmt.ExecuteQueryAsync(CancellationToken.None)); } - // ─── GetCrossReference parent-identifier filter ────────────────────────────── - // SHOW FOREIGN KEYS is scoped to the FOREIGN table only, so it returns FKs to - // every parent. GetCrossReferenceAsyncNoThrow filters the returned rows by any - // SPECIFIED (non-null) parent identifier — mirroring the JDBC reference driver - // (CrossReferenceKeysDatabricksResultSetAdapter.includeRow). ParentMatches is the - // per-field predicate; these pin the three behaviors the filter relies on: - // • null requested → no constraint (matches any row) — the GetColumnsExtended - // foreign-only reuse passes null parents and must stay UNFILTERED; - // • specified value → case-insensitive equality; - // • empty string → matches only an empty row value (no real FK has one), so - // an empty-string parent (the diff[6] case) filters every row out → empty. - // Reflection is used because the predicate is a private static helper (same - // approach as DatabricksStatementTests.GetConfOverlay). - private static bool InvokeParentMatches(string? requested, string? rowValue) + [Theory] + [InlineData((int)MetadataOperation.GetSchemas, "get_schemas")] + [InlineData((int)MetadataOperation.GetPrimaryKeys, "get_primary_keys")] + [InlineData((int)MetadataOperation.GetCrossReference, "get_cross_reference")] + public async Task MetadataRead_ObjectNotFound_ReturnsEmptyAndTraces(int operation, string name) { - var method = typeof(StatementExecutionStatement).GetMethod( - "ParentMatches", - System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); - Assert.NotNull(method); - return (bool)method!.Invoke(null, new object?[] { requested, rowValue })!; + using var activity = new Activity("metadata-test").Start(); + var exception = new DatabricksException("TABLE_OR_VIEW_NOT_FOUND"); + + int result = await StatementExecutionConnection.ReadMetadataOrEmptyAsync( + (MetadataOperation)operation, () => Task.FromException(exception), () => 0); + + Assert.Equal(0, result); + var traceEvent = Assert.Single(activity.Events); + Assert.Equal($"statement.{name}.object_not_found", traceEvent.Name); + Assert.Equal(exception.Message, Assert.Single(traceEvent.Tags).Value); } [Theory] @@ -761,27 +759,21 @@ private static bool InvokeParentMatches(string? requested, string? rowValue) [InlineData(null, "")] // null requested → matches even empty row value [InlineData("comparator_tests", "comparator_tests")] // exact match [InlineData("COMPARATOR_TESTS", "comparator_tests")] // case-insensitive match - public void ParentMatches_NullOrEqual_ReturnsTrue(string? requested, string rowValue) + public void ExactCatalogFilter_NullOrEqual_ReturnsTrue(string? requested, string rowValue) { - Assert.True(InvokeParentMatches(requested, rowValue)); + Assert.True(MetadataRowReader.MatchesCatalog(CatalogFilter.Exact(requested), rowValue)); } [Theory] [InlineData("", "comparator_tests")] // empty-string requested (diff[6]) → filters out [InlineData("other_catalog", "comparator_tests")] // specified, non-matching → filters out [InlineData("comparator_tests", null)] // specified vs NULL server column → filters out - public void ParentMatches_SpecifiedNonMatching_ReturnsFalse(string requested, string? rowValue) + public void ExactCatalogFilter_SpecifiedNonMatching_ReturnsFalse(string requested, string? rowValue) { - Assert.False(InvokeParentMatches(requested, rowValue)); + Assert.False(MetadataRowReader.MatchesCatalog(CatalogFilter.Exact(requested), rowValue)); } - // ─── GetCrossReference parent-filter WIRING (end-to-end at the Http seam) ───── - // The tests above pin the ParentMatches predicate in isolation. These exercise the - // full SEA row-filtering loop in GetCrossReferenceAsyncNoThrow: a mocked SHOW FOREIGN - // KEYS result carrying FKs to TWO different parents flows through server-value - // extraction → ParentMatches × 3 → continue → value population. A regression that - // filtered on the wrong variable, or that let the empty-parent case (diff[6]) leak - // rows instead of collapsing to zero, would pass the predicate suite but fail here. + // Exercise parent filtering through the HTTP metadata path, not just the predicate. // A SHOW FOREIGN KEYS result with two rows: orders(customer_id) → main.sales.customers // and orders(manager_id) → main.hr.employees. Both share the same FOREIGN table diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs new file mode 100644 index 00000000..0bd3f4e0 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/StatementExecutionMetadataResponseTests.cs @@ -0,0 +1,499 @@ +/* + * Copyright (c) 2026 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using AdbcDrivers.HiveServer2.Spark; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Adbc.Tests.Metadata; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Moq; +using Moq.Protected; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class StatementExecutionMetadataResponseTests + { + private static readonly string[] s_catalogs = { "main", "marketing", "other", "foo_bar", "fooxbar" }; + + public static IEnumerable CatalogPatterns() + { + (string? Pattern, string[] Matches)[] patterns = + { + ("main", new[] { "main" }), + ("ma%", new[] { "main", "marketing" }), + ("ma_n", new[] { "main" }), + (null, s_catalogs), + ("%", s_catalogs), + ("%%", s_catalogs), + ("", System.Array.Empty()), + ("missing%", System.Array.Empty()), + ("MA%", new[] { "main", "marketing" }), + (@"foo\_bar", new[] { "foo_bar" }), + ("foo_bar", new[] { "foo_bar", "fooxbar" }), + }; + foreach (int mode in new[] { 0, 1 }) + { + foreach (var depth in new[] { AdbcConnection.GetObjectsDepth.All, AdbcConnection.GetObjectsDepth.Tables }) + foreach (var (pattern, matches) in patterns) + yield return new object?[] { mode, depth, pattern, matches }; + } + foreach (int mode in new[] { 2, 3 }) + yield return new object?[] { mode, AdbcConnection.GetObjectsDepth.All, null, s_catalogs }; + } + + [Theory] + [MemberData(nameof(CatalogPatterns))] + public async Task GetObjects_NativeCatalogPatternsPreserveChildrenAndShowScopes( + int mode, AdbcConnection.GetObjectsDepth depth, string? pattern, string[] expectedCatalogs) + { + List statements = new List(); + using HttpClient http = CreateHttpClient(mode, statements, expectedCatalogs); + using StatementExecutionConnection connection = CreateConnection(http); + using IArrowArrayStream stream = connection.GetObjects(depth, pattern, "default", "t1", null, null); + using RecordBatch? batch = await stream.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + List catalogs = GetObjectsParser.ParseCatalog(batch, null); + Assert.Equal(expectedCatalogs.OrderBy(name => name), + catalogs.Select(catalog => catalog.Name).OrderBy(name => name)); + foreach (AdbcCatalog catalog in catalogs) + { + if (mode == 0 && pattern != null && + !string.Equals(pattern, catalog.Name, StringComparison.OrdinalIgnoreCase)) + { + Assert.Empty(catalog.DbSchemas!); + continue; + } + + AdbcDbSchema schema = Assert.Single(catalog.DbSchemas!); + Assert.Equal("default", schema.Name); + AdbcTable table = Assert.Single(schema.Tables!); + Assert.Equal("t1", table.Name); + if (depth == AdbcConnection.GetObjectsDepth.All) + { + AdbcColumn column = Assert.Single(table.Columns!); + Assert.Equal("a", column.Name); + Assert.Equal(1, column.OrdinalPosition); + } + } + + int catalogQueries = pattern == null && depth == AdbcConnection.GetObjectsDepth.All ? 2 : 1; + Assert.Equal(catalogQueries, + statements.Count(sql => sql.StartsWith("SHOW CATALOGS", StringComparison.Ordinal))); + Assert.Single(statements, sql => sql.StartsWith("SHOW SCHEMAS", StringComparison.Ordinal)); + Assert.Single(statements, sql => sql.StartsWith("SHOW TABLES", StringComparison.Ordinal)); + int columnQueries = depth != AdbcConnection.GetObjectsDepth.All ? 0 + : pattern == null ? s_catalogs.Length : 1; + Assert.Equal(columnQueries, + statements.Count(sql => sql.StartsWith("SHOW COLUMNS", StringComparison.Ordinal))); + if (pattern != null) + { + string scope = $"`{pattern.ToLowerInvariant()}`"; + Assert.Contains(statements, sql => sql.StartsWith("SHOW SCHEMAS", StringComparison.Ordinal) + && sql.Contains(scope)); + Assert.Contains(statements, sql => sql.StartsWith("SHOW TABLES", StringComparison.Ordinal) + && sql.Contains(scope)); + if (depth == AdbcConnection.GetObjectsDepth.All) + Assert.Contains(statements, sql => sql.StartsWith("SHOW COLUMNS", StringComparison.Ordinal) + && sql.Contains(scope)); + } + } + + [Theory] + [InlineData("GetCatalogs", false)] + [InlineData("GetCatalogs", true)] + [InlineData("GetSchemas", false)] + [InlineData("GetSchemas", true)] + [InlineData("GetTables", false)] + [InlineData("GetTables", true)] + [InlineData("GetColumns", false)] + [InlineData("GetColumns", true)] + public void MetadataCommand_RecordsResponseFormat(string command, bool native) + { + using HttpClient http = CreateHttpClient(native ? 1 : 0, new List()); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SetOption(ApacheParameters.SchemaName, "default"); + statement.SetOption(ApacheParameters.TableName, "t1"); + statement.SqlQuery = command; + QueryResult result = statement.ExecuteQuery(); + using IArrowArrayStream? stream = result.Stream; + + Assert.Equal(native, statement.IsNativeMetadataResult); + } + + [Fact] + public void MetadataCommand_ResetsNativeFlagForSyntheticResults() + { + using HttpClient http = CreateHttpClient(1, new List()); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SqlQuery = "GetTables"; + using IArrowArrayStream? native = statement.ExecuteQuery().Stream; + Assert.True(statement.IsNativeMetadataResult); + + statement.SetOption(ApacheParameters.TableTypes, ""); + using IArrowArrayStream? empty = statement.ExecuteQuery().Stream; + Assert.False(statement.IsNativeMetadataResult); + } + + [Theory] + [InlineData("main", false, "main")] + [InlineData("ma%", false, "main,marketing")] + [InlineData("MA%", false, "main,marketing")] + [InlineData("ma_n", false, "main")] + [InlineData("foo_bar", false, "foo_bar,fooxbar")] + [InlineData(@"foo\_bar", false, "foo_bar")] + [InlineData("ma%", true, "")] + [InlineData("foo_bar", true, "foo_bar")] + [InlineData("%", true, "")] + public async Task GetTables_NativeCatalogPatternsRespectLiteralMode( + string catalog, bool literal, string expectedCatalogs) + { + var statements = new List(); + using HttpClient http = CreateHttpClient(1, statements); + using StatementExecutionConnection connection = CreateConnection(http); + using var statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.CatalogName, catalog); + statement.SetOption(ApacheParameters.EscapePatternWildcards, literal.ToString()); + statement.SqlQuery = "GetTables"; + using var stream = statement.ExecuteQuery().Stream!; + using var batch = (await stream.ReadNextRecordBatchAsync())!; + + Assert.True(statement.IsNativeMetadataResult); + Assert.Equal(expectedCatalogs.Length == 0 ? System.Array.Empty() : expectedCatalogs.Split(','), + Enumerable.Range(0, batch.Length).Select(row => ((StringArray)batch.Column("TABLE_CAT")).GetString(row))); + Assert.Contains($"IN CATALOG `{catalog}`", Assert.Single(statements)); + } + + [Theory] + [InlineData("main")] + [InlineData("marketing")] + public async Task GetColumns_MixedResponses_PreserveNativeAttributesAndFlatOrdinals(string nativeCatalog) + { + using HttpClient http = CreateHttpClient(3, new List(), + nativeColumnsCatalog: nativeCatalog, columnType: "STRUCT"); + using StatementExecutionConnection connection = CreateConnection(http); + using StatementExecutionStatement statement = (StatementExecutionStatement)connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SqlQuery = "GetColumns"; + using IArrowArrayStream stream = statement.ExecuteQuery().Stream!; + using RecordBatch batch = (await stream.ReadNextRecordBatchAsync())!; + + Assert.False(statement.IsNativeMetadataResult); + Assert.Equal(s_catalogs.Length, batch.Length); + var catalogs = (StringArray)batch.Column("TABLE_CAT"); + var ordinals = (Int32Array)batch.Column("ORDINAL_POSITION"); + var defaults = (StringArray)batch.Column("COLUMN_DEF"); + var autoIncrement = (StringArray)batch.Column("IS_AUTO_INCREMENT"); + Assert.Equal(s_catalogs, Enumerable.Range(0, batch.Length).Select(row => catalogs.GetString(row))); + + using HttpClient nativeHttp = CreateHttpClient(1, new List(), columnType: "STRUCT"); + using var nativeConnection = CreateConnection(nativeHttp); + using var nativeStatement = nativeConnection.CreateStatement(); + nativeStatement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + nativeStatement.SetOption(ApacheParameters.CatalogName, nativeCatalog); + nativeStatement.SetOption(ApacheParameters.EscapePatternWildcards, "true"); + nativeStatement.SqlQuery = "GetColumns"; + using var nativeStream = nativeStatement.ExecuteQuery().Stream!; + using var nativeBatch = (await nativeStream.ReadNextRecordBatchAsync())!; + Assert.Equal(1, nativeBatch.Length); + + for (int row = 0; row < batch.Length; row++) + { + Assert.Equal(0, ordinals.GetValue(row)); + bool native = catalogs.GetString(row) == nativeCatalog; + Assert.Equal(native ? "7" : null, defaults.GetString(row)); + Assert.Equal(native ? "YES" : "NO", autoIncrement.GetString(row)); + Assert.Equal(native ? 4 : 0, ((Int32Array)batch.Column("COLUMN_SIZE")).GetValue(row)); + Assert.Equal(native ? "server comment" : "", ((StringArray)batch.Column("REMARKS")).GetString(row)); + if (!native) continue; + for (int column = 0; column < batch.ColumnCount; column++) + { + switch (batch.Column(column)) + { + case StringArray strings: + Assert.Equal(((StringArray)nativeBatch.Column(column)).GetString(0), strings.GetString(row)); + break; + case Int16Array numbers: + Assert.Equal(((Int16Array)nativeBatch.Column(column)).GetValue(0), numbers.GetValue(row)); + break; + case Int32Array numbers: + Assert.Equal(((Int32Array)nativeBatch.Column(column)).GetValue(0), numbers.GetValue(row)); + break; + default: + throw new InvalidOperationException($"Unexpected metadata type: {batch.Column(column).Data.DataType}"); + } + } + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GetObjects_SkippedShowColumnsDoNotChangeOrdinals(bool native) + { + using HttpClient http = CreateHttpClient( + native ? 1 : 0, new List(), new[] { "main" }, prependEmptyShowColumn: true); + using StatementExecutionConnection connection = CreateConnection(http); + using IArrowArrayStream stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, "main", "default", "t1", null, null); + using RecordBatch batch = (await stream.ReadNextRecordBatchAsync())!; + var catalog = Assert.Single(GetObjectsParser.ParseCatalog(batch, null)); + var schema = Assert.Single(catalog.DbSchemas!); + var table = Assert.Single(schema.Tables!); + var column = Assert.Single(table.Columns!); + Assert.Equal("a", column.Name); + Assert.Equal(1, column.OrdinalPosition); + + using var statement = connection.CreateStatement(); + statement.SetOption(ApacheParameters.IsMetadataCommand, "true"); + statement.SetOption(ApacheParameters.EscapePatternWildcards, "true"); + statement.SetOption(ApacheParameters.CatalogName, "main"); + statement.SetOption(ApacheParameters.SchemaName, "default"); + statement.SetOption(ApacheParameters.TableName, "t1"); + statement.SqlQuery = "GetColumns"; + using var flatStream = statement.ExecuteQuery().Stream!; + using var flatBatch = (await flatStream.ReadNextRecordBatchAsync())!; + Assert.Equal(native ? 1 : 2, flatBatch.Length); + Assert.Equal(native ? "a" : "", ((StringArray)flatBatch.Column("COLUMN_NAME")).GetString(0)); + Assert.Equal(0, ((Int32Array)flatBatch.Column("ORDINAL_POSITION")).GetValue(0)); + if (!native) + Assert.Equal(1, ((Int32Array)flatBatch.Column("ORDINAL_POSITION")).GetValue(1)); + } + + [Theory] + [InlineData(0, false, false)] + [InlineData(1, false, true)] + [InlineData(0, true, false)] + [InlineData(1, true, true)] + [InlineData(2, true, false)] + [InlineData(3, true, false)] + public void NativeResponseAssertions_CheckInternalMetadataStatements( + int mode, bool getObjects, bool expectedNative) + { + using HttpClient http = CreateHttpClient(mode, new List()); + using var connection = (MetadataRecordingConnection)CreateConnection(http, recordMetadata: true); + string[] commands; + if (getObjects) + { + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.All, null, "default", "t1", null, null); + commands = new[] { "SHOW CATALOGS", "SHOW SCHEMAS", "SHOW TABLES", "SHOW COLUMNS" }; + } + else + { + Assert.NotEmpty(connection.GetTableSchema("main", "default", "t1").FieldsList); + commands = new[] { "SHOW COLUMNS" }; + } + + if (expectedNative) + { + connection.AssertNativeResponses(commands); + } + else + { + var exception = Assert.ThrowsAny( + () => connection.AssertNativeResponses(commands)); + Assert.Contains("fell back to SHOW", exception.Message); + } + } + + private static StatementExecutionConnection CreateConnection(HttpClient http, bool recordMetadata = false) + { + Dictionary properties = new Dictionary + { + [SparkParameters.HostName] = "test.databricks.com", + [DatabricksParameters.WarehouseId] = "wh-1", + [SparkParameters.AccessToken] = "token", + }; + return recordMetadata + ? new MetadataRecordingConnection(properties, http) + : new StatementExecutionConnection(properties, http); + } + + private static HttpClient CreateHttpClient( + int mode, List statements, string[]? matchingCatalogs = null, bool prependEmptyShowColumn = false, + string nativeColumnsCatalog = "main", string columnType = "INT") + { + Mock handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => + { + if (request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true) + return Response("{\"session_id\":\"s1\"}"); + if (request.Method != HttpMethod.Post) + return new HttpResponseMessage(HttpStatusCode.OK); + + Assert.Equal("true", Assert.Single(request.Headers.GetValues("x-databricks-sea-can-run-fully-sync"))); + Assert.Equal("true", Assert.Single(request.Headers.GetValues("x-databricks-require-thrift-native-metadata"))); + using JsonDocument json = JsonDocument.Parse( + request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); + string sql = json.RootElement.GetProperty("statement").GetString()!; + statements.Add(sql); + string operation = request.Headers.GetValues("x-databricks-metadata-operation-type").Single(); + bool allCatalogs = sql.Contains("IN ALL CATALOGS") || operation == "GetCatalogs"; + string? requestedCatalog = allCatalogs + ? null : Regex.Match(sql, @" IN (?:CATALOG )?`((?:``|[^`])*)`").Groups[1].Value.Replace("``", "`"); + bool native = mode == 1 || (mode == 2 && operation is "GetCatalogs" or "GetTables") || + (mode == 3 && (operation != "GetColumns" || requestedCatalog == nativeColumnsCatalog)); + if (!native && !allCatalogs && !s_catalogs.Contains(requestedCatalog)) + { + return Response(JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new + { + state = "FAILED", + sql_state = "42704", + error = new { error_code = "CATALOG_NOT_FOUND", message = "Catalog not found" }, + }, + })); + } + string[] catalogs = native || requestedCatalog == null ? s_catalogs : new[] { requestedCatalog }; + if (!native && operation == "GetCatalogs") + catalogs = matchingCatalogs ?? s_catalogs; + + using RecordBatch batch = CreateBatch(operation, native, catalogs, requestedCatalog, columnType: columnType); + using MemoryStream raw = new MemoryStream(); + using (ArrowStreamWriter writer = new ArrowStreamWriter(raw, batch.Schema)) + { + if (!native && operation == "GetColumns" && prependEmptyShowColumn) + { + using var empty = CreateBatch(operation, false, catalogs, requestedCatalog, + columnName: "", columnType: columnType); + writer.WriteRecordBatch(empty); + } + writer.WriteRecordBatch(batch); + writer.WriteEnd(); + } + return Response(JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = native, + total_row_count = batch.Length, + schema = new + { + column_count = batch.ColumnCount, + columns = batch.Schema.FieldsList.Select((field, position) => new + { + name = field.Name, + position, + type_name = field.DataType.TypeId == ArrowTypeId.String ? "STRING" + : field.DataType.TypeId == ArrowTypeId.Int16 ? "SMALLINT" : "INT", + }).ToArray(), + }, + }, + result = new { attachment = raw.ToArray() }, + })); + }); + return new HttpClient(handler.Object); + } + + private static HttpResponseMessage Response(string body) + => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + + private static RecordBatch CreateBatch( + string operation, bool native, string[] catalogs, string? requestedCatalog, + string columnName = "a", string columnType = "INT") + { + Schema schema; + if (native) + { + schema = operation switch + { + "GetCatalogs" => MetadataSchemaFactory.CreateCatalogsSchema(), + "GetSchemas" => MetadataSchemaFactory.CreateSchemasSchema(), + "GetTables" => MetadataSchemaFactory.CreateTablesSchema(), + _ => new Schema(MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23), null), + }; + } + else + { + string[] names = operation switch + { + "GetCatalogs" => new[] { "catalog" }, + "GetSchemas" => requestedCatalog == null ? new[] { "databaseName", "catalog" } : new[] { "databaseName" }, + "GetTables" => new[] { "catalogName", "namespace", "tableName", "tableType" }, + _ => new[] { "catalogName", "namespace", "tableName", "col_name", "columnType", "isNullable" }, + }; + schema = new Schema(names.Select(name => new Field(name, StringType.Default, true)), null); + } + + IArrowArray[] arrays = schema.FieldsList.Select(field => + { + if (field.DataType.TypeId == ArrowTypeId.String) + { + StringArray.Builder strings = new StringArray.Builder(); + foreach (string catalog in catalogs) + { + string? value = field.Name switch + { + "TABLE_CAT" or "TABLE_CATALOG" or "catalog" or "catalogName" => catalog, + "TABLE_SCHEM" or "databaseName" or "namespace" => "default", + "TABLE_NAME" or "tableName" => "t1", + "TABLE_TYPE" or "tableType" => "TABLE", + "COLUMN_NAME" or "col_name" => columnName, + "TYPE_NAME" or "columnType" => columnType, + "COLUMN_DEF" => "7", + "REMARKS" => "server comment", + "IS_AUTO_INCREMENT" => "YES", + "isNullable" => "true", + _ => null, + }; + if (value == null) strings.AppendNull(); else strings.Append(value); + } + return (IArrowArray)strings.Build(); + } + int number = field.Name == "NULLABLE" ? 1 : field.Name is "DATA_TYPE" or "COLUMN_SIZE" ? 4 : 0; + return field.DataType.TypeId switch + { + ArrowTypeId.Int16 => (IArrowArray)new Int16Array.Builder().AppendRange( + Enumerable.Repeat((short)number, catalogs.Length)).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().AppendRange( + Enumerable.Repeat(number, catalogs.Length)).Build(), + _ => throw new InvalidOperationException($"Unexpected metadata type: {field.DataType}"), + }; + }).ToArray(); + return new RecordBatch(schema, arrays, catalogs.Length); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs new file mode 100644 index 00000000..54aac053 --- /dev/null +++ b/csharp/test/Unit/StatementExecution/StatementExecutionNativeCatalogTests.cs @@ -0,0 +1,277 @@ +/* + * Copyright (c) 2025 ADBC Drivers Contributors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using AdbcDrivers.Databricks.StatementExecution; +using AdbcDrivers.HiveServer2; +using AdbcDrivers.HiveServer2.Hive2; +using AdbcDrivers.HiveServer2.Spark; +using Apache.Arrow; +using Apache.Arrow.Adbc; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using Moq; +using Moq.Protected; +using Xunit; + +namespace AdbcDrivers.Databricks.Tests.Unit.StatementExecution +{ + public class StatementExecutionNativeCatalogTests + { + private static byte[] ArrowStrings(string name, params string[] values) + { + var schema = new Schema(new[] { new Field(name, StringType.Default, true) }, null); + var builder = new StringArray.Builder(); + foreach (string value in values) builder.Append(value); + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, new IArrowArray[] { builder.Build() }, values.Length)); + writer.WriteEnd(); + } + return raw.ToArray(); + } + + private static byte[] ArrowNativeColumns(Field[] fields, string nativeCatalog) + { + var schema = new Schema(fields, null); + var arrays = fields.Select(field => + { + if (field.Name == "TABLE_CAT") + return (IArrowArray)new StringArray.Builder().Append(nativeCatalog).Append("fooxbar").Build(); + string value = field.Name switch + { + "TABLE_SCHEM" => "default", + "TABLE_NAME" => "t", + "COLUMN_NAME" => "a", + "TYPE_NAME" => "INT", + "IS_AUTO_INCREMENT" => "YES", + _ => field.Name, + }; + return field.DataType.TypeId switch + { + ArrowTypeId.String => (IArrowArray)new StringArray.Builder() + .Append(value).Append(value).Build(), + ArrowTypeId.Int8 => new Int8Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int16 => new Int16Array.Builder().Append(0).Append(0).Build(), + ArrowTypeId.Int32 => new Int32Array.Builder().Append(0).Append(0).Build(), + _ => (IArrowArray)new Int64Array.Builder().Append(0).Append(0).Build(), + }; + }).ToArray(); + using var raw = new MemoryStream(); + using (var writer = new ArrowStreamWriter(raw, schema)) + { + writer.WriteRecordBatch(new RecordBatch(schema, arrays, 2)); + writer.WriteEnd(); + } + return raw.ToArray(); + } + + private static HttpClient CreateHttpClient( + bool overlappingNativeColumns = false, Field[]? columnFields = null, string nativeCatalog = "foo_bar") + { + byte[] catalogs = overlappingNativeColumns + ? ArrowStrings("TABLE_CAT", nativeCatalog, "fooxbar") + : ArrowStrings("TABLE_CAT", "main", "other"); + Field[] nativeColumnFields = columnFields ?? + MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); + byte[] columns = overlappingNativeColumns + ? ArrowNativeColumns(nativeColumnFields, nativeCatalog) + : ArrowStrings("col_name", "a"); + var handler = new Mock(); + handler.Protected() + .Setup>("SendAsync", + ItExpr.IsAny(), ItExpr.IsAny()) + .ReturnsAsync((HttpRequestMessage request, CancellationToken _) => + { + if (request.RequestUri?.AbsolutePath.EndsWith("/sql/sessions") == true) + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent("{\"session_id\":\"s1\"}") }; + + string sql = ""; + if (request.Method == HttpMethod.Post) + { + using var json = JsonDocument.Parse(request.Content!.ReadAsStringAsync().GetAwaiter().GetResult()); + sql = json.RootElement.GetProperty("statement").GetString() ?? ""; + } + bool isCatalogs = sql.StartsWith("SHOW CATALOGS"); + Field[] fields = isCatalogs + ? new[] { new Field("TABLE_CAT", StringType.Default, true) } + : overlappingNativeColumns + ? nativeColumnFields + : new[] { new Field("col_name", StringType.Default, true) }; + var manifestColumns = fields.Select((field, position) => + { + string type = field.DataType.TypeId switch + { + ArrowTypeId.Int8 => "TINYINT", + ArrowTypeId.Int16 => "SMALLINT", + ArrowTypeId.Int32 => "INT", + ArrowTypeId.Int64 => "BIGINT", + _ => "STRING", + }; + return new { name = field.Name, position, type_name = type, type_text = type }; + }).ToArray(); + var body = JsonSerializer.Serialize(new + { + statement_id = "stmt-1", + status = new { state = "SUCCEEDED" }, + manifest = new + { + is_native_metadata_result = isCatalogs || overlappingNativeColumns, + total_row_count = isCatalogs || overlappingNativeColumns ? 2 : 1, + schema = new + { + column_count = fields.Length, + columns = manifestColumns, + }, + }, + result = new { attachment = isCatalogs ? catalogs : columns }, + }); + return new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) }; + }); + return new HttpClient(handler.Object); + } + + private static StatementExecutionConnection CreateConnection(HttpClient http) + { + var properties = new Dictionary + { + [SparkParameters.HostName] = "test.databricks.com", + [DatabricksParameters.WarehouseId] = "wh-1", + [SparkParameters.AccessToken] = "token", + }; + return new StatementExecutionConnection(properties, http); + } + + [Fact] + public async Task GetObjects_CatalogDepth_FiltersNativeCatalogs() + { + using var http = CreateHttpClient(); + using var connection = CreateConnection(http); + using var stream = connection.GetObjects( + AdbcConnection.GetObjectsDepth.Catalogs, "main", null, null, null, null); + using var batch = await stream.ReadNextRecordBatchAsync(); + + Assert.NotNull(batch); + Assert.Equal(1, batch.Length); + Assert.Equal("main", ((StringArray)batch.Column(0)).GetString(0)); + } + + [Fact] + public async Task ColumnFanout_KeepsTheQueriedCatalog() + { + using var http = CreateHttpClient(); + using var connection = CreateConnection(http); + + var result = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); + + Assert.Equal(2, result.Results.Count); + Assert.Equal(new[] { "main", "other" }, result.Results.Select(response => response.SourceCatalog)); + } + + [Theory] + [InlineData("count", "expected 23 columns, found 22")] + [InlineData("name", "missing TABLE_CAT")] + [InlineData("type", "unexpected type or duplicate TABLE_SCHEM")] + public async Task ColumnFanout_MalformedNativeColumns_PropagatesValidationError( + string invalidSchema, string expectedError) + { + Field[] fields = MetadataSchemaFactory.CreateColumnMetadataSchema().FieldsList.Take(23).ToArray(); + switch (invalidSchema) + { + case "count": + fields = fields.Skip(1).ToArray(); + break; + case "name": + fields[0] = new Field("UNKNOWN", fields[0].DataType, true); + break; + case "type": + fields[1] = new Field(fields[1].Name, Int32Type.Default, true); + break; + } + using var http = CreateHttpClient(overlappingNativeColumns: true, columnFields: fields); + using var connection = CreateConnection(http); + + var exception = await Assert.ThrowsAsync(() => + connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None)); + + Assert.StartsWith("Invalid native GetColumns result:", exception.Message); + Assert.Contains(expectedError, exception.Message); + } + + [Theory] + [InlineData("foo_bar", "foo_bar", 1)] + [InlineData("FOO_BAR", "foo_bar", 1)] + [InlineData("foo%bar", "foo%bar", 1)] + [InlineData("FOO%BAR", "foo%bar", 1)] + [InlineData("fooxbar", "foo_bar", 1)] + [InlineData("missing", "foo_bar", 0)] + [InlineData(null, "foo_bar", 2)] + [InlineData("SPARK", "foo_bar", 2)] + public void GetTableSchema_UsesExactNativeCatalogScope( + string? requestedCatalog, string nativeCatalog, int expectedFields) + { + using var http = CreateHttpClient(overlappingNativeColumns: true, nativeCatalog: nativeCatalog); + using var connection = CreateConnection(http); + + var schema = connection.GetTableSchema(requestedCatalog, "default", "t"); + + Assert.Equal(expectedFields, schema.FieldsList.Count); + Assert.All(schema.FieldsList, field => Assert.Equal("a", field.Name)); + } + + [Fact] + public async Task ColumnFanout_FiltersOverlappingNativeCatalogNames() + { + using var http = CreateHttpClient(overlappingNativeColumns: true); + using var connection = CreateConnection(http); + + var columns = await connection.ReadColumnsAsync(null, null, null, null, CancellationToken.None); + + Assert.True(columns.IsNative); + Assert.Equal(2, columns.Results.Count); + Assert.All(columns.Results, response => Assert.Equal(2, Assert.Single(response.Batches).Length)); + Assert.Equal(new[] { "foo_bar", "fooxbar" }, columns.Rows.Select(row => row.Catalog)); + + var result = NativeMetadataResultBuilder.Build( + columns.Results, + MetadataSchemaFactory.CreateColumnMetadataSchema(), MetadataOperation.GetColumns); + using var stream = result.Stream!; + Assert.Equal(2, result.RowCount); + + var catalogMap = new Dictionary>>(); + foreach (string catalog in new[] { "foo_bar", "fooxbar" }) + { + catalogMap[catalog] = new Dictionary> + { + ["default"] = new Dictionary { ["t"] = new TableInfo("TABLE") }, + }; + } + await ((IGetObjectsDataProvider)connection).PopulateColumnInfoAsync( + null, null, null, null, catalogMap, CancellationToken.None); + Assert.Single(catalogMap["foo_bar"]["default"]["t"].ColumnName); + Assert.Single(catalogMap["fooxbar"]["default"]["t"].ColumnName); + Assert.True(catalogMap["foo_bar"]["default"]["t"].IsAutoIncrement.Single()); + } + } +} diff --git a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs index 90dd298d..4181eaf3 100644 --- a/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs +++ b/csharp/test/Unit/StatementExecution/StatementExecutionPollingFailedStateTests.cs @@ -160,6 +160,48 @@ public async Task ExecuteUpdateAsync_FailedAfterPolling_ThrowsDatabricksExceptio Assert.Contains("table not found", exception.Message); } + [Theory] + [InlineData(true, null, true)] + [InlineData(null, true, true)] + [InlineData(true, false, false)] + [InlineData(null, false, false)] + [InlineData(null, null, false)] + public async Task MetadataResultAfterPolling_UsesManifestMarker( + bool? initialFlag, bool? finalFlag, bool expectedNative) + { + _mockClient.Setup(c => c.ExecuteStatementAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new ExecuteStatementResponse + { + StatementId = StatementId, + Status = new StatementStatus { State = "PENDING" }, + Manifest = initialFlag.HasValue + ? new ResultManifest { IsNativeMetadataResult = initialFlag } + : null, + }); + _mockClient.Setup(c => c.GetStatementAsync(StatementId, It.IsAny())) + .ReturnsAsync(new GetStatementResponse + { + StatementId = StatementId, + Status = new StatementStatus { State = "SUCCEEDED" }, + Manifest = new ResultManifest + { + IsNativeMetadataResult = finalFlag, + Schema = new ResultSchema + { + ColumnCount = 1, + Columns = new List { new ColumnInfo { Name = "TABLE_CAT", Position = 0, TypeName = "STRING" } }, + }, + }, + }); + + using var stmt = CreateStatement(); + var queryResult = await stmt.ExecuteQueryAsync( + CancellationToken.None, isMetadataExecution: true, metadataOperation: MetadataOperation.GetCatalogs); + using var stream = queryResult.Stream; + + Assert.Equal(expectedNative, stmt.IsNativeMetadataResult); + } + public void Dispose() { _httpClient?.Dispose();