Skip to content

Commit 7f49541

Browse files
authored
feat(session): register object_store backends on SessionContextBuilder (#73)
1 parent 77b5b72 commit 7f49541

11 files changed

Lines changed: 1983 additions & 16 deletions

File tree

Lines changed: 334 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,334 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.datafusion;
21+
22+
import org.apache.datafusion.protobuf.GcsOptions;
23+
import org.apache.datafusion.protobuf.HttpOptions;
24+
import org.apache.datafusion.protobuf.ObjectStoreRegistration;
25+
import org.apache.datafusion.protobuf.S3Options;
26+
27+
/**
28+
* Description of an {@code object_store::ObjectStore} backend to register on a {@link
29+
* SessionContext}'s {@code RuntimeEnv}. Pass instances to {@link
30+
* SessionContextBuilder#registerObjectStore(ObjectStoreOptions)}.
31+
*
32+
* <p>Build instances via the per-backend factories: {@link #s3()}, {@link #gcs()}, {@link
33+
* #http(String)}. Each factory returns a builder whose {@code build()} produces an immutable {@code
34+
* ObjectStoreOptions} you can register.
35+
*
36+
* <p>For the cloud backends (S3 / GCS), the standard SDK environment variables are read as the
37+
* default <em>floor</em> (e.g. {@code AWS_ACCESS_KEY_ID}, {@code AWS_DEFAULT_REGION}, ECS /
38+
* web-identity vars; {@code GOOGLE_APPLICATION_CREDENTIALS}). Explicit setters on the per-backend
39+
* builder always override those env values, so a JVM running on a credentialed host (IAM role, ECS
40+
* task, GKE workload identity) can register an empty-credentials store and let the SDK's default
41+
* chain do its job.
42+
*
43+
* <p>The URL DataFusion uses to look up the store is either an explicit {@code url(...)} on the
44+
* builder or, if unset, derived from the typed fields:
45+
*
46+
* <ul>
47+
* <li>S3 → {@code s3://<bucket>}
48+
* <li>GCS → {@code gs://<bucket>}
49+
* <li>HTTP → no scheme-default; {@link #http(String)} requires the URL up front. The URL must be
50+
* a host root ({@code https://example.com/} or {@code https://example.com}); paths cause
51+
* lookup-vs-base-URL collisions in DataFusion's registry. Register one HTTP store per
52+
* scheme+host and let SQL paths carry the rest.
53+
* </ul>
54+
*/
55+
public abstract sealed class ObjectStoreOptions
56+
permits ObjectStoreOptions.S3, ObjectStoreOptions.Gcs, ObjectStoreOptions.Http {
57+
58+
private final String url;
59+
60+
private ObjectStoreOptions(String url) {
61+
this.url = url;
62+
}
63+
64+
/** Begin building an {@link S3} (also covers MinIO, R2, any S3-compatible) registration. */
65+
public static S3.Builder s3() {
66+
return new S3.Builder();
67+
}
68+
69+
/** Begin building a {@link Gcs} (Google Cloud Storage) registration. */
70+
public static Gcs.Builder gcs() {
71+
return new Gcs.Builder();
72+
}
73+
74+
/**
75+
* Begin building an {@link Http} (listing-capable HTTP/WebDAV) registration. The listing-root URL
76+
* is required because no sensible default exists.
77+
*
78+
* @throws IllegalArgumentException if {@code listingUrl} is {@code null}.
79+
*/
80+
public static Http.Builder http(String listingUrl) {
81+
if (listingUrl == null) {
82+
throw new IllegalArgumentException("http listing URL must be non-null");
83+
}
84+
return new Http.Builder(listingUrl);
85+
}
86+
87+
abstract ObjectStoreRegistration toRegistration();
88+
89+
/** Amazon S3 (and any S3-compatible endpoint such as MinIO, Cloudflare R2, Wasabi). */
90+
public static final class S3 extends ObjectStoreOptions {
91+
private final String bucket;
92+
private final String region;
93+
private final String endpoint;
94+
private final String accessKeyId;
95+
private final String secretAccessKey;
96+
private final String sessionToken;
97+
private final Boolean allowHttp;
98+
private final Boolean skipSignature;
99+
private final Boolean imdsv1Fallback;
100+
101+
private S3(Builder b) {
102+
super(b.url);
103+
this.bucket = b.bucket;
104+
this.region = b.region;
105+
this.endpoint = b.endpoint;
106+
this.accessKeyId = b.accessKeyId;
107+
this.secretAccessKey = b.secretAccessKey;
108+
this.sessionToken = b.sessionToken;
109+
this.allowHttp = b.allowHttp;
110+
this.skipSignature = b.skipSignature;
111+
this.imdsv1Fallback = b.imdsv1Fallback;
112+
}
113+
114+
@Override
115+
ObjectStoreRegistration toRegistration() {
116+
S3Options.Builder s3 = S3Options.newBuilder().setBucket(bucket);
117+
if (region != null) s3.setRegion(region);
118+
if (endpoint != null) s3.setEndpoint(endpoint);
119+
if (accessKeyId != null) s3.setAccessKeyId(accessKeyId);
120+
if (secretAccessKey != null) s3.setSecretAccessKey(secretAccessKey);
121+
if (sessionToken != null) s3.setSessionToken(sessionToken);
122+
if (allowHttp != null) s3.setAllowHttp(allowHttp);
123+
if (skipSignature != null) s3.setSkipSignature(skipSignature);
124+
if (imdsv1Fallback != null) s3.setImdsv1Fallback(imdsv1Fallback);
125+
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setS3(s3);
126+
if (super.url != null) reg.setUrl(super.url);
127+
return reg.build();
128+
}
129+
130+
public static final class Builder {
131+
private String url;
132+
private String bucket;
133+
private String region;
134+
private String endpoint;
135+
private String accessKeyId;
136+
private String secretAccessKey;
137+
private String sessionToken;
138+
private Boolean allowHttp;
139+
private Boolean skipSignature;
140+
private Boolean imdsv1Fallback;
141+
142+
private Builder() {}
143+
144+
/**
145+
* Override the registration URL. Default is {@code s3://<bucket>}; use this to register under
146+
* a non-default scheme such as {@code s3a://}.
147+
*/
148+
public Builder url(String url) {
149+
this.url = url;
150+
return this;
151+
}
152+
153+
/** Bucket name. Required. */
154+
public Builder bucket(String bucket) {
155+
this.bucket = bucket;
156+
return this;
157+
}
158+
159+
public Builder region(String region) {
160+
this.region = region;
161+
return this;
162+
}
163+
164+
/** Override the endpoint URL (e.g. {@code https://minio.internal:9000}). */
165+
public Builder endpoint(String endpoint) {
166+
this.endpoint = endpoint;
167+
return this;
168+
}
169+
170+
public Builder accessKeyId(String accessKeyId) {
171+
this.accessKeyId = accessKeyId;
172+
return this;
173+
}
174+
175+
public Builder secretAccessKey(String secretAccessKey) {
176+
this.secretAccessKey = secretAccessKey;
177+
return this;
178+
}
179+
180+
public Builder sessionToken(String sessionToken) {
181+
this.sessionToken = sessionToken;
182+
return this;
183+
}
184+
185+
/** Allow plain {@code http://} endpoints (e.g. local MinIO). */
186+
public Builder allowHttp(boolean allowHttp) {
187+
this.allowHttp = allowHttp;
188+
return this;
189+
}
190+
191+
/** Skip request signing (anonymous / public-bucket reads). */
192+
public Builder skipSignature(boolean skipSignature) {
193+
this.skipSignature = skipSignature;
194+
return this;
195+
}
196+
197+
/** Allow falling back to IMDSv1 for credential discovery on EC2. */
198+
public Builder imdsv1Fallback(boolean imdsv1Fallback) {
199+
this.imdsv1Fallback = imdsv1Fallback;
200+
return this;
201+
}
202+
203+
public S3 build() {
204+
if (bucket == null || bucket.isEmpty()) {
205+
throw new IllegalArgumentException("S3 ObjectStoreOptions: bucket is required");
206+
}
207+
return new S3(this);
208+
}
209+
}
210+
}
211+
212+
/** Google Cloud Storage. */
213+
public static final class Gcs extends ObjectStoreOptions {
214+
private final String bucket;
215+
private final String serviceAccountKey;
216+
private final String serviceAccountPath;
217+
private final String applicationCredentials;
218+
219+
private Gcs(Builder b) {
220+
super(b.url);
221+
this.bucket = b.bucket;
222+
this.serviceAccountKey = b.serviceAccountKey;
223+
this.serviceAccountPath = b.serviceAccountPath;
224+
this.applicationCredentials = b.applicationCredentials;
225+
}
226+
227+
@Override
228+
ObjectStoreRegistration toRegistration() {
229+
GcsOptions.Builder gcs = GcsOptions.newBuilder().setBucket(bucket);
230+
if (serviceAccountKey != null) gcs.setServiceAccountKey(serviceAccountKey);
231+
if (serviceAccountPath != null) gcs.setServiceAccountPath(serviceAccountPath);
232+
if (applicationCredentials != null) gcs.setApplicationCredentials(applicationCredentials);
233+
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setGcs(gcs);
234+
if (super.url != null) reg.setUrl(super.url);
235+
return reg.build();
236+
}
237+
238+
public static final class Builder {
239+
private String url;
240+
private String bucket;
241+
private String serviceAccountKey;
242+
private String serviceAccountPath;
243+
private String applicationCredentials;
244+
245+
private Builder() {}
246+
247+
/** Override the registration URL. Default is {@code gs://<bucket>}. */
248+
public Builder url(String url) {
249+
this.url = url;
250+
return this;
251+
}
252+
253+
/** Bucket name. Required. */
254+
public Builder bucket(String bucket) {
255+
this.bucket = bucket;
256+
return this;
257+
}
258+
259+
/** Inline service-account JSON. Mutually exclusive with {@link #serviceAccountPath}. */
260+
public Builder serviceAccountKey(String serviceAccountKey) {
261+
this.serviceAccountKey = serviceAccountKey;
262+
return this;
263+
}
264+
265+
/**
266+
* Filesystem path to service-account JSON. Mutually exclusive with {@link
267+
* #serviceAccountKey}.
268+
*/
269+
public Builder serviceAccountPath(String serviceAccountPath) {
270+
this.serviceAccountPath = serviceAccountPath;
271+
return this;
272+
}
273+
274+
/**
275+
* Filesystem path to the application-default-credentials JSON. Maps to {@code
276+
* with_application_credentials} on the upstream Rust builder, which takes a path, not inline
277+
* content.
278+
*/
279+
public Builder applicationCredentials(String applicationCredentialsPath) {
280+
this.applicationCredentials = applicationCredentialsPath;
281+
return this;
282+
}
283+
284+
public Gcs build() {
285+
if (bucket == null || bucket.isEmpty()) {
286+
throw new IllegalArgumentException("GCS ObjectStoreOptions: bucket is required");
287+
}
288+
if (serviceAccountKey != null && serviceAccountPath != null) {
289+
throw new IllegalArgumentException(
290+
"GCS ObjectStoreOptions: serviceAccountKey and serviceAccountPath are mutually"
291+
+ " exclusive");
292+
}
293+
return new Gcs(this);
294+
}
295+
}
296+
}
297+
298+
/** Listing-capable HTTP / WebDAV store. */
299+
public static final class Http extends ObjectStoreOptions {
300+
private final Boolean allowHttp;
301+
302+
private Http(Builder b) {
303+
super(b.url);
304+
this.allowHttp = b.allowHttp;
305+
}
306+
307+
@Override
308+
ObjectStoreRegistration toRegistration() {
309+
HttpOptions.Builder http = HttpOptions.newBuilder();
310+
if (allowHttp != null) http.setAllowHttp(allowHttp);
311+
ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setHttp(http);
312+
reg.setUrl(super.url);
313+
return reg.build();
314+
}
315+
316+
public static final class Builder {
317+
private final String url;
318+
private Boolean allowHttp;
319+
320+
private Builder(String url) {
321+
this.url = url;
322+
}
323+
324+
public Builder allowHttp(boolean allowHttp) {
325+
this.allowHttp = allowHttp;
326+
return this;
327+
}
328+
329+
public Http build() {
330+
return new Http(this);
331+
}
332+
}
333+
}
334+
}

‎core/src/main/java/org/apache/datafusion/SessionContextBuilder.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@
1919

2020
package org.apache.datafusion;
2121

22+
import java.util.ArrayList;
2223
import java.util.LinkedHashMap;
24+
import java.util.List;
2325
import java.util.Map;
2426

2527
import org.apache.datafusion.protobuf.ConfigOption;
@@ -39,6 +41,7 @@ public final class SessionContextBuilder {
3941
private Double memoryLimitFraction;
4042
private String tempDirectory;
4143
private final LinkedHashMap<String, String> options = new LinkedHashMap<>();
44+
private final List<ObjectStoreOptions> objectStores = new ArrayList<>();
4245

4346
SessionContextBuilder() {}
4447

@@ -168,6 +171,31 @@ public SessionContextBuilder setOptions(Map<String, String> entries) {
168171
return this;
169172
}
170173

174+
/**
175+
* Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}.
176+
* Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3},
177+
* {@link ObjectStoreOptions#gcs}, {@link ObjectStoreOptions#http(String)}). The store is
178+
* reachable inside the resulting {@link SessionContext} by URL — e.g. once an S3 store is
179+
* registered for {@code my-bucket}, {@code ctx.registerParquet("orders",
180+
* "s3://my-bucket/orders/")} will resolve through it.
181+
*
182+
* <p>Multiple registrations are applied in the order added; if two registrations resolve to the
183+
* same URL, the later one wins (matching upstream {@code RuntimeEnv::register_object_store}).
184+
*
185+
* <p>If the underlying {@code object_store} cloud-backend Cargo feature is not built into the
186+
* native library, {@link #build} surfaces a clear error rather than silently dropping the
187+
* registration. The default {@code make} build enables all three backends (S3 / GCS / HTTP).
188+
*
189+
* @throws IllegalArgumentException if {@code options} is {@code null}.
190+
*/
191+
public SessionContextBuilder registerObjectStore(ObjectStoreOptions options) {
192+
if (options == null) {
193+
throw new IllegalArgumentException("registerObjectStore options must be non-null");
194+
}
195+
this.objectStores.add(options);
196+
return this;
197+
}
198+
171199
/**
172200
* Construct a {@link SessionContext} with the configured options.
173201
*
@@ -204,6 +232,9 @@ byte[] toBytes() {
204232
for (Map.Entry<String, String> e : options.entrySet()) {
205233
b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build());
206234
}
235+
for (ObjectStoreOptions os : objectStores) {
236+
b.addObjectStores(os.toRegistration());
237+
}
207238
return b.build().toByteArray();
208239
}
209240
}

0 commit comments

Comments
 (0)