From 9dc3cd13ed524e85002651e28584812052148cec Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 15:04:35 +0100 Subject: [PATCH 01/10] Enabled analyzers --- .../Microsoft.Azure.Databricks.Client.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index 31e1dc6..188b24c 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -1,5 +1,6 @@ + true net6.0;net8.0;net9.0 true false From 7f488a540bcf4808676a3d275fe3bc18002e781b Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 15:11:25 +0100 Subject: [PATCH 02/10] Added Directory.Build.props + other pre-existing solution items --- csharp/Directory.Build.props | 5 +++++ .../Microsoft.Azure.Databricks.Client.Sample.csproj | 2 -- .../Microsoft.Azure.Databricks.Client.Test.csproj | 2 -- csharp/Microsoft.Azure.Databricks.Client.sln | 3 +++ .../Microsoft.Azure.Databricks.Client.csproj | 1 - 5 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 csharp/Directory.Build.props diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props new file mode 100644 index 0000000..9268d6b --- /dev/null +++ b/csharp/Directory.Build.props @@ -0,0 +1,5 @@ + + + net6.0;net8.0;net9.0 + + diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj index 3590294..8aca520 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj @@ -1,8 +1,6 @@  - Exe - net6.0;net8.0;net9.0 latest false enable diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index 31933a9..b716e15 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -1,7 +1,5 @@  - - net6.0;net8.0;net9.0 latest false enable diff --git a/csharp/Microsoft.Azure.Databricks.Client.sln b/csharp/Microsoft.Azure.Databricks.Client.sln index 3eeee5d..0226ef1 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.sln +++ b/csharp/Microsoft.Azure.Databricks.Client.sln @@ -12,6 +12,9 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{0D112210-7B29-455F-B677-6491F870228B}" ProjectSection(SolutionItems) = preProject .editorconfig = .editorconfig + Directory.Build.props = Directory.Build.props + Directory.Packages.props = Directory.Packages.props + nuget.config = nuget.config EndProjectSection EndProject Global diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index 188b24c..8d2a124 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -1,7 +1,6 @@ true - net6.0;net8.0;net9.0 true false true From f4fd9dafa8528f7e656e18666d2adc2dc3a21bda Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 16:33:49 +0100 Subject: [PATCH 03/10] Remove .Net 6 as target framework due to EOL. --- csharp/Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/Directory.Build.props b/csharp/Directory.Build.props index 9268d6b..bbe4a9a 100644 --- a/csharp/Directory.Build.props +++ b/csharp/Directory.Build.props @@ -1,5 +1,5 @@ - net6.0;net8.0;net9.0 + net8.0;net9.0 From 2259bd3158f2da36771cb17ad827da7687e53c12 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 16:36:11 +0100 Subject: [PATCH 04/10] Updated StatementExectionApiClient Execute, Get & GetResultChunk to use source generation. --- .../ApiClient.cs | 60 ++++++- .../DatabricksSerialisationContext.cs | 24 +++ .../ISQLApi.cs | 167 +++++++++--------- .../StatementExecutionApiClient.cs | 109 ++++++------ 4 files changed, 218 insertions(+), 142 deletions(-) create mode 100644 csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs diff --git a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs index 3743fe5..f09d545 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs @@ -1,18 +1,21 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - -using Microsoft.Azure.Databricks.Client.Converters; +#nullable enable using System; +using System.Diagnostics.CodeAnalysis; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Mime; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; using System.Threading; using System.Threading.Tasks; +using Microsoft.Azure.Databricks.Client.Converters; + namespace Microsoft.Azure.Databricks.Client; public abstract class ApiClient : IDisposable @@ -52,7 +55,16 @@ protected static ClientApiException CreateApiException(HttpResponseMessage respo return new ClientApiException(errorContent, statusCode); } - private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody body, CancellationToken cancellationToken = default) + protected static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, StringContent? content, JsonTypeInfo typeInfo, CancellationToken cancellationToken = default) + { + using var response = await FetchResponse(httpClient, method, requestUri, content, cancellationToken).ConfigureAwait(false); + await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + + return await JsonSerializer.DeserializeAsync(responseStream, typeInfo, cancellationToken).ConfigureAwait(false); + } + + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody? body, CancellationToken cancellationToken = default) { using var response = await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); @@ -60,18 +72,21 @@ private static async Task SendRequest(HttpClient httpCl return await JsonSerializer.DeserializeAsync(responseStream, Options, cancellationToken).ConfigureAwait(false); } - private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody body, CancellationToken cancellationToken = default) + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + private static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, TBody? body, CancellationToken cancellationToken = default) { await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] private static async Task SendHeadRequest(HttpClient httpClient, HttpMethod method, - string requestUri, TBody body = default, CancellationToken cancellationToken = default) + string requestUri, TBody? body = default, CancellationToken cancellationToken = default) { using var response = await FetchResponse(httpClient, method, requestUri, body, cancellationToken).ConfigureAwait(false); return response.Content.Headers; } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] private static async Task FetchResponse(HttpClient httpClient, HttpMethod method, string requestUri, TBody body, CancellationToken cancellationToken = default) { @@ -90,54 +105,81 @@ private static async Task FetchResponse(HttpClient h return response; } - protected static async Task HttpGet(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) + private static async Task FetchResponse(HttpClient httpClient, HttpMethod method, + string requestUri, StringContent? content, CancellationToken cancellationToken = default) + { + var request = new HttpRequestMessage(method, requestUri) + { + Content = content, + }; + + var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + { + throw CreateApiException(response); + } + + return response; + } + + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + protected static async Task HttpGet(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) { return await SendRequest(httpClient, HttpMethod.Get, requestUri, null, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpPost(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { await SendRequest(httpClient, HttpMethod.Post, requestUri, body, cancellationToken).ConfigureAwait(false); } - protected static async Task HttpPost(HttpClient httpClient, string requestUri, + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + protected static async Task HttpPost(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { return await SendRequest(httpClient, HttpMethod.Post, requestUri, body, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpPatch(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { await SendRequest(httpClient, HttpMethod.Patch, requestUri, body, cancellationToken).ConfigureAwait(false); } - protected static async Task HttpPatch(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + protected static async Task HttpPatch(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { return await SendRequest(httpClient, HttpMethod.Patch, requestUri, body, cancellationToken).ConfigureAwait(false); } - protected static async Task HttpPut(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + protected static async Task HttpPut(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { return await SendRequest(httpClient, HttpMethod.Put, requestUri, body, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpPut(HttpClient httpClient, string requestUri, TBody body, CancellationToken cancellationToken = default) { await SendRequest(httpClient, HttpMethod.Put, requestUri, body, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpDelete(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) { await SendRequest(httpClient, HttpMethod.Delete, requestUri, null, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpHead(HttpClient httpClient, string requestUri, TBody body = default, CancellationToken cancellationToken = default) { return await SendHeadRequest(httpClient, HttpMethod.Head, requestUri, body, cancellationToken).ConfigureAwait(false); } + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] protected static async Task HttpHead(HttpClient httpClient, string requestUri, CancellationToken cancellationToken = default) { return await SendHeadRequest(httpClient, HttpMethod.Head, requestUri, null, cancellationToken).ConfigureAwait(false); diff --git a/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs b/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs new file mode 100644 index 0000000..70c955c --- /dev/null +++ b/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs @@ -0,0 +1,24 @@ +using System.Text.Json.Serialization; + +using Microsoft.Azure.Databricks.Client.Converters; +using Microsoft.Azure.Databricks.Client.Models; + +namespace Microsoft.Azure.Databricks.Client; + +[JsonSerializable(typeof(SqlStatement))] +[JsonSerializable(typeof(StatementExecution))] +[JsonSerializable(typeof(StatementExecutionResultChunk))] +[JsonSourceGenerationOptions( + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingDefault, + PropertyNameCaseInsensitive = true, + Converters = new System.Type[] { + typeof(JsonStringEnumConverter), + typeof(MillisecondEpochDateTimeConverter), + typeof(LibraryConverter), + typeof(SecretScopeConverter), + typeof(AclPermissionItemConverter), + typeof(DepedencyConverter), + typeof(TableConstraintConverter), + } +)] +internal sealed partial class DatabricksSerializationContext : JsonSerializerContext { } diff --git a/csharp/Microsoft.Azure.Databricks.Client/ISQLApi.cs b/csharp/Microsoft.Azure.Databricks.Client/ISQLApi.cs index c77489e..edc26d4 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/ISQLApi.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/ISQLApi.cs @@ -1,92 +1,95 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -using Microsoft.Azure.Databricks.Client.Models; using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Azure.Databricks.Client +using Microsoft.Azure.Databricks.Client.Models; + +namespace Microsoft.Azure.Databricks.Client; + +public interface ISQLApi : IDisposable +{ + IStatementExecutionApi StatementExecution { get; } + + IWarehouseApi Warehouse { get; } +} + +public interface IWarehouseApi : IDisposable +{ + /// + /// Creates a new SQL warehouse. + /// + Task Create(WarehouseAttributes warehouseAttributes, CancellationToken cancellationToken = default); + + /// + /// Lists all SQL warehouses that a user has manager permissions on. + /// + /// Service Principal which will be used to fetch the list of warehouses. + /// If not specified, the user from the session header is used. + Task> List(int? runAsUserId = default, CancellationToken cancellationToken = default); + + /// + /// Gets the information for a single SQL warehouse. + /// + /// Required. Id of the SQL warehouse. + Task Get(string id, CancellationToken cancellationToken = default); + + /// + /// Deletes a SQL warehouse. + /// + /// Required. Id of the SQL warehouse. + Task Delete(string id, CancellationToken cancellationToken = default); + + /// + /// Updates the configuration for a SQL warehouse. + /// + /// Required. Id of the warehouse to configure. + Task Update(string id, WarehouseAttributes warehouseAttributes, CancellationToken cancellationToken = default); + + /// + /// Starts a SQL warehouse. + /// + /// Required. Id of the SQL warehouse. + Task Start(string id, CancellationToken cancellationToken = default); + + /// + /// Stops a SQL warehouse. + /// + /// Required. Id of the SQL warehouse. + Task Stop(string id, CancellationToken cancellationToken = default); +} + +public interface IStatementExecutionApi : IDisposable { - public interface ISQLApi : IDisposable - { - IStatementExecutionApi StatementExecution { get; } - IWarehouseApi Warehouse { get; } - } - - public interface IWarehouseApi : IDisposable - { - /// - /// Creates a new SQL warehouse. - /// - Task Create(WarehouseAttributes warehouseAttributes, CancellationToken cancellationToken = default); - - /// - /// Lists all SQL warehouses that a user has manager permissions on. - /// - /// Service Principal which will be used to fetch the list of warehouses. - /// If not specified, the user from the session header is used. - Task> List(int? runAsUserId = default, CancellationToken cancellationToken = default); - - /// - /// Gets the information for a single SQL warehouse. - /// - /// Required. Id of the SQL warehouse. - Task Get(string id, CancellationToken cancellationToken = default); - - /// - /// Deletes a SQL warehouse. - /// - /// Required. Id of the SQL warehouse. - Task Delete(string id, CancellationToken cancellationToken = default); - - /// - /// Updates the configuration for a SQL warehouse. - /// - /// Required. Id of the warehouse to configure. - Task Update(string id, WarehouseAttributes warehouseAttributes, CancellationToken cancellationToken = default); - - /// - /// Starts a SQL warehouse. - /// - /// Required. Id of the SQL warehouse. - Task Start(string id, CancellationToken cancellationToken = default); - - /// - /// Stops a SQL warehouse. - /// - /// Required. Id of the SQL warehouse. - Task Stop(string id, CancellationToken cancellationToken = default); - } - - public interface IStatementExecutionApi : IDisposable - { - /// - /// Execute a SQL statement. - /// - Task Execute(SqlStatement statement, CancellationToken cancellationToken = default); - - /// - /// Cancel statement execution. - /// - /// Requried. Id of statement execution. - Task Cancel(string id, CancellationToken cancellationToken = default); - - /// - /// Get status, manifest, and result first chunk. - /// - /// Requried. Id of statement execution. - Task Get(string id, CancellationToken cancellationToken = default); - - /// - /// Get result chunk by index. - /// - /// - /// After the statement execution has SUCCEEDED, this request can be used to fetch any chunk by index. Whereas the first chunk with chunk_index=0 is typically fetched with statementexecution/executestatement or statementexecution/getstatement, this request can be used to fetch subsequent chunks. The response structure is identical to the nested result element described in the statementexecution/getstatement request, and similarly includes the next_chunk_index and next_chunk_internal_link fields for simple iteration through the result set. - /// - /// Requried. Id of statement execution. - /// Required. The index of the chunk. - Task GetResultChunk(string id, int chunkIndex, CancellationToken cancellationToken = default); - } + /// + /// Execute a SQL statement. + /// + Task Execute(SqlStatement statement, CancellationToken cancellationToken = default); + + /// + /// Cancel statement execution. + /// + /// Required. Id of statement execution. + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + Task Cancel(string id, CancellationToken cancellationToken = default); + + /// + /// Get status, manifest, and result first chunk. + /// + /// Required. Id of statement execution. + Task Get(string id, CancellationToken cancellationToken = default); + + /// + /// Get result chunk by index. + /// + /// + /// After the statement execution has SUCCEEDED, this request can be used to fetch any chunk by index. Whereas the first chunk with chunk_index=0 is typically fetched with statementexecution/executestatement or statementexecution/getstatement, this request can be used to fetch subsequent chunks. The response structure is identical to the nested result element described in the statementexecution/getstatement request, and similarly includes the next_chunk_index and next_chunk_internal_link fields for simple iteration through the result set. + /// + /// Required. Id of statement execution. + /// Required. The index of the chunk. + Task GetResultChunk(string id, int chunkIndex, CancellationToken cancellationToken = default); } diff --git a/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs index 95985f8..a45932b 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs @@ -1,63 +1,70 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#nullable enable -using Microsoft.Azure.Databricks.Client.Models; +using System.Diagnostics.CodeAnalysis; using System.Net.Http; +using System.Net.Mime; +using System.Text; using System.Text.Json; -using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; -namespace Microsoft.Azure.Databricks.Client +using Microsoft.Azure.Databricks.Client.Models; + +namespace Microsoft.Azure.Databricks.Client; + +public class StatementExecutionApiClient : ApiClient, IStatementExecutionApi { - public class StatementExecutionApiClient : ApiClient, IStatementExecutionApi + private readonly string _apiBaseUrl; + + public StatementExecutionApiClient(HttpClient httpClient) : base(httpClient) + { + _apiBaseUrl = $"{ApiVersion}/sql/statements"; + } + + [RequiresUnreferencedCode("Calls System.Text.Json.JsonSerializer.DeserializeAsync(Stream, JsonSerializerOptions, CancellationToken)")] + public async Task Cancel(string id, CancellationToken cancellationToken = default) + { + await HttpPost(this.HttpClient, $"{this._apiBaseUrl}/{id}/cancel", new { }, cancellationToken).ConfigureAwait(false); + } + + public async Task Execute(SqlStatement statement, CancellationToken cancellationToken = default) + { + var request = JsonSerializer.Serialize(statement, DatabricksSerializationContext.Default.SqlStatement); + var body = new StringContent(request, Encoding.UTF8, MediaTypeNames.Application.Json); + + return await SendRequest( + this.HttpClient, + HttpMethod.Post, + this._apiBaseUrl, + body, + DatabricksSerializationContext.Default.StatementExecution, + cancellationToken + ).ConfigureAwait(false); + } + + public async Task GetResultChunk(string id, int chunkIndex, CancellationToken cancellationToken = default) + { + return await SendRequest( + this.HttpClient, + HttpMethod.Get, + $"{this._apiBaseUrl}/{id}/result/chunks/{chunkIndex}", + null, + DatabricksSerializationContext.Default.StatementExecutionResultChunk, + cancellationToken + ).ConfigureAwait(false); + } + + public async Task Get(string id, CancellationToken cancellationToken = default) { - private readonly string _apiBaseUrl; - - public StatementExecutionApiClient(HttpClient httpClient) : base(httpClient) - { - _apiBaseUrl = $"{ApiVersion}/sql/statements"; - } - - public async Task Cancel(string id, CancellationToken cancellationToken = default) - { - await HttpPost(this.HttpClient, $"{this._apiBaseUrl}/{id}/cancel", new { }, cancellationToken).ConfigureAwait(false); - } - - public async Task Execute(SqlStatement statement, CancellationToken cancellationToken = default) - { - var jsonObj = JsonSerializer.SerializeToNode(statement, Options)!.AsObject(); - - var execution = await HttpPost( - this.HttpClient, - this._apiBaseUrl, - jsonObj, - cancellationToken - ).ConfigureAwait(false); - - return execution.Deserialize(Options); - } - - public async Task GetResultChunk(string id, int chunkIndex, CancellationToken cancellationToken = default) - { - var execution = await HttpGet( - this.HttpClient, - $"{this._apiBaseUrl}/{id}/result/chunks/{chunkIndex}", - cancellationToken - ).ConfigureAwait(false); - - return execution.Deserialize(Options); - } - - public async Task Get(string id, CancellationToken cancellationToken = default) - { - var execution = await HttpGet( - this.HttpClient, - $"{this._apiBaseUrl}/{id}", - cancellationToken - ).ConfigureAwait(false); - - return execution.Deserialize(Options); - } + return await SendRequest( + this.HttpClient, + HttpMethod.Get, + $"{this._apiBaseUrl}/{id}", + null, + DatabricksSerializationContext.Default.StatementExecution, + cancellationToken + ).ConfigureAwait(false); } } From 76b1370a22a0b5c8042074ea30c8df044a4fe054 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 16:49:58 +0100 Subject: [PATCH 05/10] Remove old package override --- .../Microsoft.Azure.Databricks.Client.Sample.csproj | 3 --- .../Microsoft.Azure.Databricks.Client.Test.csproj | 3 --- .../Microsoft.Azure.Databricks.Client.csproj | 3 --- 3 files changed, 9 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj index 8aca520..d315238 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Sample/Microsoft.Azure.Databricks.Client.Sample.csproj @@ -12,7 +12,4 @@ - - - diff --git a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj index b716e15..1007976 100644 --- a/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client.Test/Microsoft.Azure.Databricks.Client.Test.csproj @@ -21,7 +21,4 @@ - - - diff --git a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj index 8d2a124..c14615f 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj +++ b/csharp/Microsoft.Azure.Databricks.Client/Microsoft.Azure.Databricks.Client.csproj @@ -33,7 +33,4 @@ - - - From 12ef120fae1df494e69d61faf7cb91a766aef6dd Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Thu, 16 Oct 2025 16:58:52 +0100 Subject: [PATCH 06/10] Removal of .Net 6 allows primary constructor --- .../DatabricksSerialisationContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs b/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs index 70c955c..6d85469 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/DatabricksSerialisationContext.cs @@ -21,4 +21,4 @@ namespace Microsoft.Azure.Databricks.Client; typeof(TableConstraintConverter), } )] -internal sealed partial class DatabricksSerializationContext : JsonSerializerContext { } +internal sealed partial class DatabricksSerializationContext : JsonSerializerContext; From c3cbd3e5f3a90e02fd0f25595a42ae4d8d4ae4d8 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Fri, 17 Oct 2025 09:46:05 +0100 Subject: [PATCH 07/10] Use JsonContent instead of StringContent to avoid use of string (UTF-16) when the content is JSON (UTF-8). --- csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs | 4 ++-- .../StatementExecutionApiClient.cs | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs index f09d545..a30a81b 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/ApiClient.cs @@ -55,7 +55,7 @@ protected static ClientApiException CreateApiException(HttpResponseMessage respo return new ClientApiException(errorContent, statusCode); } - protected static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, StringContent? content, JsonTypeInfo typeInfo, CancellationToken cancellationToken = default) + protected static async Task SendRequest(HttpClient httpClient, HttpMethod method, string requestUri, HttpContent? content, JsonTypeInfo typeInfo, CancellationToken cancellationToken = default) { using var response = await FetchResponse(httpClient, method, requestUri, content, cancellationToken).ConfigureAwait(false); await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); @@ -106,7 +106,7 @@ private static async Task FetchResponse(HttpClient h } private static async Task FetchResponse(HttpClient httpClient, HttpMethod method, - string requestUri, StringContent? content, CancellationToken cancellationToken = default) + string requestUri, HttpContent? content, CancellationToken cancellationToken = default) { var request = new HttpRequestMessage(method, requestUri) { diff --git a/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs b/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs index a45932b..e64eb28 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/StatementExecutionApiClient.cs @@ -4,9 +4,9 @@ using System.Diagnostics.CodeAnalysis; using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Net.Mime; -using System.Text; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -31,14 +31,12 @@ public async Task Cancel(string id, CancellationToken cancellationToken = defaul public async Task Execute(SqlStatement statement, CancellationToken cancellationToken = default) { - var request = JsonSerializer.Serialize(statement, DatabricksSerializationContext.Default.SqlStatement); - var body = new StringContent(request, Encoding.UTF8, MediaTypeNames.Application.Json); - + var content = JsonContent.Create(statement, DatabricksSerializationContext.Default.SqlStatement, new MediaTypeHeaderValue(MediaTypeNames.Application.Json)); return await SendRequest( this.HttpClient, HttpMethod.Post, this._apiBaseUrl, - body, + content, DatabricksSerializationContext.Default.StatementExecution, cancellationToken ).ConfigureAwait(false); From af32125fcad68df65fd939103d5d5a2eb5a50ffa Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Fri, 17 Oct 2025 10:41:32 +0100 Subject: [PATCH 08/10] Additional RequiresUnreferencedCode usage for StatementExecution --- .../Models/StatementExecution.cs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs b/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs index 4668c84..1ff45c7 100644 --- a/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs +++ b/csharp/Microsoft.Azure.Databricks.Client/Models/StatementExecution.cs @@ -1,11 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. - using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; -using System.Text.Json.Serialization; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; namespace Microsoft.Azure.Databricks.Client.Models; @@ -71,7 +71,7 @@ public static SqlStatement Create(string statement, string warehouseId, params S public long? RowLimit { get; set; } /// - /// Applies the given byte limit to the statement's result size. Byte counts are based on internal data representations and might not match the final size in the requested format. If the result was truncated due to the byte limit, then truncated in the response is set to true. When using EXTERNAL_LINKS disposition, a default byte_limit of 100 GiB is applied if byte_limit is not explcitly set. + /// Applies the given byte limit to the statement's result size. Byte counts are based on internal data representations and might not match the final size in the requested format. If the result was truncated due to the byte limit, then truncated in the response is set to true. When using EXTERNAL_LINKS disposition, a default byte_limit of 100 GiB is applied if byte_limit is not explicitly set. /// [JsonPropertyName("byte_limit")] public long? ByteLimit { get; set; } @@ -139,7 +139,7 @@ public enum SqlStatementDisposition /// /// Statements executed with EXTERNAL_LINKS disposition will return result data as external links: URLs that point to cloud storage internal to the workspace. Using EXTERNAL_LINKS disposition allows statements to generate arbitrarily sized result sets for fetching up to 100 GiB. The resulting links have two important properties: /// 1. They point to resources external to the Databricks compute; therefore any associated authentication information (typically a personal access token, OAuth token, or similar) must be removed when fetching from these links. - /// 2. These are presigned URLs with a specific expiration, indicated in the response. The behavior when attempting to use an expired link is cloud specific. + /// 2. These are pre-signed URLs with a specific expiration, indicated in the response. The behavior when attempting to use an expired link is cloud specific. /// EXTERNAL_LINKS } @@ -282,11 +282,12 @@ public record StatementExecution public StatementExecutionManifest Manifest { get; set; } /// - /// Contains the result data of a single chunk when using INLINE disposition. When using EXTERNAL_LINKS disposition, the array external_links is used instead to provide presigned URLs to the result data in cloud storage. Exactly one of these alternatives is used. (While the external_links array prepares the API to return multiple links in a single response. Currently only a single link is returned.) + /// Contains the result data of a single chunk when using INLINE disposition. When using EXTERNAL_LINKS disposition, the array external_links is used instead to provide pre-signed URLs to the result data in cloud storage. Exactly one of these alternatives is used. (While the external_links array prepares the API to return multiple links in a single response. Currently only a single link is returned.) /// [JsonPropertyName("result")] public StatementExecutionResultChunk Result { get; set; } + [RequiresUnreferencedCode("Uses System.Text.Json.Nodes.JsonArray.Add.")] public IEnumerable DeserializeResults(Func rowFactory) { if (this.Manifest.Format == StatementFormat.JSON_ARRAY) @@ -449,6 +450,7 @@ public override int GetHashCode() { hash *= chunk.GetHashCode(); } + return hash; } } @@ -481,6 +483,7 @@ public override int GetHashCode() { hash *= column.GetHashCode(); } + return hash; } } @@ -606,6 +609,7 @@ public record StatementExecutionResultChunk : StatementExecutionResult [JsonIgnore] public JsonArray DataJsonArray { + [RequiresUnreferencedCode("Uses System.Text.Json.Nodes.JsonArray.Add().")] get { var jsonArray = new JsonArray(); @@ -654,7 +658,7 @@ public override int GetHashCode() public record StatementExecutionExternalLink : StatementExecutionResult { /// - /// A presigned URL pointing to a chunk of result data, hosted by an external service, with a short expiration time (<= 15 minutes). As this URL contains a temporary credential, it should be considered sensitive and the client should not expose this URL in a log. + /// A pre-signed URL pointing to a chunk of result data, hosted by an external service, with a short expiration time (<= 15 minutes). As this URL contains a temporary credential, it should be considered sensitive and the client should not expose this URL in a log. /// [JsonPropertyName("external_link")] public string ExternalLink { get; set; } From 645e59ceac5f631d8aeb6b89a59cb703b14ed22a Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Wed, 22 Oct 2025 10:11:52 +0100 Subject: [PATCH 09/10] Removed .Net6 reference from workflow --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6b66c39..1bf376d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,6 @@ jobs: dotnet-version: | 9.x 8.x - 6.x - name: Restore run: dotnet restore "$Solution_Name" From 7be11e6b4280c7df887fea07191fe54c1187af87 Mon Sep 17 00:00:00 2001 From: Michael Ellis Date: Wed, 22 Oct 2025 10:12:04 +0100 Subject: [PATCH 10/10] Remove .Net 6 conditional include from packages.props --- csharp/Directory.Packages.props | 3 --- 1 file changed, 3 deletions(-) diff --git a/csharp/Directory.Packages.props b/csharp/Directory.Packages.props index 3f0c48f..db16dc3 100644 --- a/csharp/Directory.Packages.props +++ b/csharp/Directory.Packages.props @@ -15,7 +15,4 @@ - - -