diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index f13101deeeb..a0ade0efe82 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -626,6 +626,7 @@ jobs: org.apache.comet.rules.CometScanRuleSuite org.apache.comet.rules.CometScanContribSuite org.apache.comet.rules.CometScanSchemeFallbackSuite + org.apache.comet.cloud.s3.CometS3CredentialReleaseSuite org.apache.comet.rules.CometMultiStoreScanSuite org.apache.comet.rules.CometExecRuleSuite org.apache.spark.sql.comet.CometPlanEqualitySuite diff --git a/.github/workflows/pr_build_macos.yml b/.github/workflows/pr_build_macos.yml index ee13480830e..ab3498d06d6 100644 --- a/.github/workflows/pr_build_macos.yml +++ b/.github/workflows/pr_build_macos.yml @@ -264,6 +264,7 @@ jobs: org.apache.comet.rules.CometScanRuleSuite org.apache.comet.rules.CometScanContribSuite org.apache.comet.rules.CometScanSchemeFallbackSuite + org.apache.comet.cloud.s3.CometS3CredentialReleaseSuite org.apache.comet.rules.CometMultiStoreScanSuite org.apache.comet.rules.CometExecRuleSuite org.apache.spark.sql.comet.CometPlanEqualitySuite diff --git a/docs/source/contributor-guide/s3-credential-provider-design.md b/docs/source/contributor-guide/s3-credential-provider-design.md index 3d455ed04eb..0681fb253de 100644 --- a/docs/source/contributor-guide/s3-credential-provider-design.md +++ b/docs/source/contributor-guide/s3-credential-provider-design.md @@ -158,20 +158,29 @@ The Parquet path forwards the full `fs.s3a.*` config subset as `catalog_properti ## Built-in adapters -Comet ships two reference SPI implementations under `org.apache.comet.cloud.s3`, so standard provider classes that the native Rust list does not match work with a config change instead of bespoke code: +Comet ships three SPI implementations under `org.apache.comet.cloud.s3`, so standard provider classes that the native Rust list does not match work with a config change instead of bespoke code: - `HadoopS3ACredentialProviderAdapter` delegates to Hadoop S3A's own provider construction (`S3AUtils.createAWSCredentialProviderSet` on Hadoop 3.3.x, `CredentialProviderListFactory.createAWSCredentialProviderList` on 3.4+). - `AwsSdkCredentialProviderAdapter` wraps a raw AWS SDK provider named in `fs.s3a.comet.credential.adapter.class`. +- `IcebergVendedS3CredentialProvider` resolves REST-vended Iceberg credentials through Iceberg's `AwsClientProperties`, as `S3FileIO` does, and refreshes them through `client.refresh-credentials-endpoint`. It exists only in the spark-4.x source set. `CometIcebergNativeScan.withVendedCredentialProvider` selects it for the native Iceberg scan, not the writer, when the catalog returns a refresh endpoint and a `uri` and no provider is configured. -Each has a spark-3.x (SDK v1) and a spark-4.x (SDK v2) body under the same FQCN, selected by the `shims.majorVerSrc` source set, so each Comet build compiles against exactly the one AWS SDK its Hadoop line ships. The SDK and `hadoop-aws` are `provided` scope only (see the `hadoop-aws.version` property in the root `pom.xml`), so Comet does not bundle a second copy. +The two Hadoop and AWS SDK adapters each have a spark-3.x (SDK v1) and a spark-4.x (SDK v2) body under the same FQCN, selected by the `shims.majorVerSrc` source set, so each Comet build compiles against exactly the one AWS SDK its Hadoop line ships. `IcebergVendedS3CredentialProvider` has only the spark-4.x (SDK v2) body, so Spark 3.x builds don't ship it and never select it. The SDK and `hadoop-aws` are `provided` scope only (see the `hadoop-aws.version` property in the root `pom.xml`), so Comet does not bundle a second copy. ## Returns or throws, not a fall-through value The SPI returns a `CometS3Credentials` or throws. There is no sentinel "I do not know" return. Vendors that are only authoritative for some paths resolve the default AWS chain themselves for the rest and return the result. This matches the contract on every other AWS credential SPI in the JVM ecosystem (AWS SDK v1/v2, Hadoop S3A, Iceberg `VendedCredentialsProvider`). -## Lifecycle: `AutoCloseable` plus a JVM shutdown hook +## Lifecycle: `AutoCloseable`, reference counts and a JVM shutdown hook -`CometS3CredentialProvider` extends `AutoCloseable` with a default no-op `close()`. The dispatcher installs a JVM shutdown hook that iterates every cached instance and calls `close()`, swallowing per-provider exceptions so a slow or buggy vendor cannot block other providers from cleaning up. Stateless providers ignore this entirely; vendors that hold long-lived resources (HTTP clients, scheduled-refresh executors, STS connection pools) override `close()` to release them. Shutdown hooks are best-effort, so a `SIGKILL` or abrupt JVM termination skips them; vendors must not rely on `close()` for correctness, only for resource hygiene. +`CometS3CredentialProvider` extends `AutoCloseable` with a default no-op `close()`. + +Each `ensureInitialized` call takes a reference to the instance it returns. A native `CometS3CredentialBridge` holds that reference, shares it with the bridges it derives (`for_path`, `for_location`), and the last of them to be dropped gives it back with `release`. When an instance's last reference is released, the dispatcher closes it and forgets its key, so a later `ensureInitialized` creates a new instance. A reference count rather than "close when superseded" keeps an instance open while any reader still uses it. + +The last bridge can be dropped on any thread: a Tokio worker running an iceberg-rust task, or a task thread inside `execute` that just missed the `FileIO` cache. So `release` hands the close to a single dispatcher-owned daemon thread, and a `close()` that blocks (Iceberg's `RefreshingAuthManager.close()` waits up to a minute for a refresh in flight) delays neither. The shutdown hook and a concurrent last release both claim an instance by setting its count to retired, so only one of them closes it. + +On the Iceberg path, bridges live in the executor's `FileIO` cache. A REST catalog that vends new credentials for a table changes the table's FileIO properties, and so the `(FQCN, dispatchKey, catalogProperties)` key. Inserting the table's new `FileIO` evicts the ones it supersedes: same access mode, catalog and table, different properties with an earlier expiry. A read identifies the table by its metadata file's directory, since Iceberg writes every metadata file of a table into one directory (`write.metadata.path`, or `/metadata`). A write identifies it by its data location. Only properties whose `s3.session-token-expires-at-ms` is later than the cached entry's supersede it, so both must carry one. Properties can also differ for reasons that don't say which is newer, such as two sessions or tenants that share a catalog name with different tokens, OAuth credentials or tenant ids. Those entries stay side by side, so the sessions don't evict each other on every task, and a provider re-vended without an expiry is released only when the LRU bound retires its entry. A commit that keeps the properties supersedes nothing. Evicted providers are closed once the tasks reading through them finish, so a long-lived executor holds one provider per table in use, not one per credential it was ever vended. The cache's 64-entry LRU bound limits the rest. On the Parquet path the object store cache never evicts, so its providers live until shutdown. + +The dispatcher also installs a JVM shutdown hook that iterates every remaining instance and calls `close()`, swallowing per-provider exceptions so a slow or buggy vendor cannot block other providers from cleaning up. Stateless providers ignore this entirely; vendors that hold long-lived resources (HTTP clients, scheduled-refresh executors, STS connection pools) override `close()` to release them. Shutdown hooks are best-effort, so a `SIGKILL` or abrupt JVM termination skips them; vendors must not rely on `close()` for correctness, only for resource hygiene. ## Iceberg path: error message fidelity caveat diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index b58daa660e1..7e23627ba8e 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -191,7 +191,7 @@ For a custom S3-compatible endpoint, configure the catalog with the endpoint, pa --conf spark.sql.catalog.s3_cat.s3.secret-access-key=... ``` -These `s3.*` storage properties are not specific to the Hive catalog shown here. When `s3.access-key-id` / `s3.secret-access-key` are omitted, credentials come from the standard AWS chain (environment variables, instance profiles, and so on). The region is not auto-detected: when neither the catalog (`client.region` or `s3.region`) nor the executor environment (`AWS_REGION` or `AWS_DEFAULT_REGION`) supplies one, Comet uses `us-east-1`, so set it for AWS buckets in any other region. If your REST catalog vends temporary credentials, the native reader does not consume them automatically, and wiring that requires the credential provider bridge. See Iceberg's [S3 FileIO](https://iceberg.apache.org/docs/latest/aws/#s3-fileio) docs for the full property list, and [S3 Credential Providers](s3-credential-providers.md) for vended or per-request credentials. +These `s3.*` storage properties are not specific to the Hive catalog shown here. When `s3.access-key-id` / `s3.secret-access-key` are omitted, credentials come from the standard AWS chain (environment variables, instance profiles, and so on). The region is not auto-detected: when neither the catalog (`client.region` or `s3.region`) nor the executor environment (`AWS_REGION` or `AWS_DEFAULT_REGION`) supplies one, Comet uses `us-east-1`, so set it for AWS buckets in any other region. If your REST catalog vends temporary credentials and returns `client.refresh-credentials-endpoint`, the native reader on Spark 4.x refreshes them through `IcebergVendedS3CredentialProvider`, which Comet selects automatically. See Iceberg's [S3 FileIO](https://iceberg.apache.org/docs/latest/aws/#s3-fileio) docs for the full property list, and [S3 Credential Providers](s3-credential-providers.md) for vended or per-request credentials. ### HDFS diff --git a/docs/source/user-guide/latest/s3-credential-providers.md b/docs/source/user-guide/latest/s3-credential-providers.md index 23659703ac9..8c449b2f2cf 100644 --- a/docs/source/user-guide/latest/s3-credential-providers.md +++ b/docs/source/user-guide/latest/s3-credential-providers.md @@ -83,6 +83,29 @@ spark.hadoop.fs.s3a.bucket..comet.credential.adapter.class= Use `HadoopS3ACredentialProviderAdapter` unless you have a plain SDK provider not wired through S3A. Both class names are the same on every Comet build; each build automatically uses the AWS SDK its Hadoop line ships (v1 on the Spark 3.4/3.5 builds, v2 on 4.0+), so you configure one name and get the right implementation. +### `IcebergVendedS3CredentialProvider` (native Iceberg reads, Spark 4.x) + +Without a provider, the native Iceberg scan signs every request with the `s3.*` credentials the REST catalog vended when the driver loaded the table. Vended credentials are short-lived, so a scan that runs past their expiry fails with `ExpiredToken`. Iceberg's `S3FileIO`, which Spark's own Iceberg reads use, refreshes them through `client.refresh-credentials-endpoint` when the catalog returns one. + +This provider resolves credentials the way `S3FileIO` does, through Iceberg's `AwsClientProperties`. It refreshes through the catalog's endpoint when one is returned, and otherwise uses the static vended keys and reports their vended expiry. Iceberg is loaded by reflection from the thread context class loader, so `iceberg-aws` can sit on the user-jar loader. The AWS SDK v2 must be visible from Comet's loader, as for the adapters above. + +Comet selects this provider automatically for native Iceberg scans when all of these hold: + +- The FileIO properties carry `client.refresh-credentials-endpoint`, and `client.refresh-credentials-enabled` is not `false`. +- The properties carry the catalog `uri`, whether the endpoint is relative or absolute. +- No provider is configured for the catalog. +- On the driver, Comet's class loader and the thread context class loader see the same AWS SDK v2. Comet logs at `INFO` when they don't. + +An explicit setting takes precedence. An empty value turns the provider off for that catalog: + +``` +spark.sql.catalog..s3.comet.credential.provider.class= +``` + +Each instance of the provider holds its own HTTP client for the refresh endpoint. Comet closes an instance once the table's cached `FileIO` is replaced, which happens only when the catalog vends credentials whose `s3.session-token-expires-at-ms` is later than the cached ones (see [Lifecycle](#lifecycle)). Catalogs that vend S3 credentials, such as Apache Polaris and Unity Catalog, send that expiry. A catalog that rotates the `s3.*` keys without it gives each table reload another instance, and its HTTP client, until the executor's cache of 64 `FileIO`s evicts the older entries. + +Spark 3.x builds do not include this provider. On Spark 3.x, a native Iceberg scan that outlives its vended credentials fails with `ExpiredToken` unless you configure a provider of your own. The native Iceberg writer does not select it automatically: a catalog that returns `client.refresh-credentials-endpoint` makes the write fall back to Spark. + ## Enabling a bridge A bridge is activated by naming the vendor's class in a Spark config. Putting a JAR on the classpath alone has no effect; the config key must be set. @@ -210,7 +233,7 @@ public interface CometS3CredentialProvider extends AutoCloseable { CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) throws Exception; - /** Invoked from the dispatcher's JVM shutdown hook. Default is a no-op. */ + /** Invoked when Comet retires the instance, or at JVM shutdown. Default is a no-op. */ @Override default void close() throws Exception {} } ``` @@ -219,11 +242,13 @@ The class must have a public no-arg constructor. `getCredentialsForPath` may be ### Lifecycle -Comet keys provider instances by `(FQCN, dispatchKey, catalogProperties)`. The dispatch key is the Spark V2 catalog name on the Iceberg path and the S3 bucket name on the Parquet path. The first time a given key is seen on an executor, Comet reflects the class, calls `initialize(Map)` exactly once, and caches the instance for the JVM lifetime. Two catalogs sharing one provider FQCN therefore get isolated instances with their own `initialize` maps. Including `catalogProperties` in the key matters in multi-tenant JVMs (Spark Connect, Thrift Server, `SparkSession.newSession()`) where two sessions can otherwise collide on the same `(FQCN, dispatchKey)` and have the second session silently use the first session's credentials. +Comet keys provider instances by `(FQCN, dispatchKey, catalogProperties)`. The dispatch key is the Spark V2 catalog name on the Iceberg path and the S3 bucket name on the Parquet path. The first time a given key is seen on an executor, Comet reflects the class, calls `initialize(Map)` once, and keeps the instance while some reader uses it. Two catalogs sharing one provider FQCN therefore get isolated instances with their own `initialize` maps. Including `catalogProperties` in the key matters in multi-tenant JVMs (Spark Connect, Thrift Server, `SparkSession.newSession()`) where two sessions can otherwise collide on the same `(FQCN, dispatchKey)` and have the second session silently use the first session's credentials. `initialize` should be cheap and non-blocking. Defer real credential fetches (REST round-trips, STS calls) to the first `getCredentialsForPath` invocation. On the Iceberg path the supplied `catalogProperties` carries the unfiltered FileIO bag, including REST-vended fields like `credentials.uri`, OAuth tokens, and any vendor-custom keys you set on the catalog config. On the Parquet path it carries the `fs.s3a.*` config subset (including static keys such as `fs.s3a.access.key`); in Comet 1.0 this map was empty on the Parquet path. The map may contain secrets, so do not log it. -`close()` is invoked from a JVM shutdown hook installed by the dispatcher. The default no-op is fine for stateless providers. Override it to release HTTP clients, scheduled-refresh executors, or STS connection pools. Shutdown hooks are best-effort: a `SIGKILL` or abrupt JVM termination skips them, so do not depend on `close()` for correctness. +Comet calls `close()` once no reader uses an instance any more, and at JVM shutdown for instances still in use. On the Iceberg path an instance is retired when the table's next `FileIO` replaces the one built on it, and when an executor's cache of 64 `FileIO`s evicts it. A new `FileIO` replaces the old one when the catalog vends new credentials for the table and both sets carry `s3.session-token-expires-at-ms`, the new one later. New credentials without a later expiry don't replace the old `FileIO`, so its instance stays open until the 64-entry bound evicts it. A later read with the same key then gets a new instance, with its own `initialize` call. On the Parquet path Comet keeps each object store, and so its provider, for the executor's lifetime. + +So `close()` can run while the job is still running. Comet calls it at most once per instance, on a dispatcher thread rather than a reader's, and possibly while a new instance for the same key runs `initialize`. Release only the instance's own resources, never state that other instances share. A `close()` that blocks delays only the closes queued behind it. The default no-op is fine for stateless providers. Override it to release HTTP clients, scheduled-refresh executors, or STS connection pools. Shutdown hooks are best-effort: a `SIGKILL` or abrupt JVM termination skips them, so do not depend on `close()` for correctness. ### Caching, refresh, and distribution @@ -358,36 +383,9 @@ public final class MyMultiTenantProvider implements CometS3CredentialProvider { } ``` -### Reference implementation: Iceberg REST vended credentials - -For Iceberg REST catalogs that vend AWS credentials (`LoadTableResponse.credentials`), the canonical implementation wraps Iceberg's existing `VendedCredentialsProvider`: - -```java -public final class IcebergRESTVendedS3Provider implements CometS3CredentialProvider { - private volatile VendedCredentialsProvider provider; - - @Override - public void initialize(Map catalogProperties) { - this.provider = VendedCredentialsProvider.create(catalogProperties); - } - - @Override - public CometS3Credentials getCredentialsForPath(CometS3CredentialContext ctx) { - AwsCredentials c = provider.resolveCredentials(); - String token = null; - long expiresAt = 0L; - if (c instanceof AwsSessionCredentials) { - AwsSessionCredentials session = (AwsSessionCredentials) c; - token = session.sessionToken(); - // Report the vended expiry, so Comet reuses the credential until shortly before it. - expiresAt = session.expirationTime().map(Instant::toEpochMilli).orElse(0L); - } - return new CometS3Credentials(c.accessKeyId(), c.secretAccessKey(), token, expiresAt); - } -} -``` +### Iceberg REST vended credentials -`VendedCredentialsProvider` reads `credentials.uri`, the catalog endpoint, and OAuth tokens from the supplied map (Comet forwards the unfiltered FileIO bag to `initialize`), and refreshes through its own `CachedSupplier`. Refresh-near-expiry and the REST round-trip live in Iceberg. Comet only reuses the credential this returns until five minutes before the expiry it reports, so report the expiry: returning `0` turns that reuse off. Comet ships a copy of this class under `spark/src/test` as a reference; copy it into your runtime jar alongside `iceberg-aws` and AWS SDK v2. +For Iceberg REST catalogs that vend AWS credentials, use the provider Comet ships, `IcebergVendedS3CredentialProvider` (see the section on it above). Comet selects it automatically when the catalog returns a refresh endpoint. To write your own, build Iceberg's credentials provider through `AwsClientProperties.credentialsProvider`, not `VendedCredentialsProvider.create`. A REST catalog returns its refresh endpoint as `client.refresh-credentials-endpoint`, often relative to `uri`, and `AwsClientProperties` resolves it into the `credentials.uri` that `VendedCredentialsProvider` reads. Report the credential's expiry: Comet reuses a credential until five minutes before the expiry you report, and returning `0` turns that reuse off. ### Access mode diff --git a/native/core/src/cloud/s3/credential_bridge.rs b/native/core/src/cloud/s3/credential_bridge.rs index 5d5c8d3f800..64d7c49b967 100644 --- a/native/core/src/cloud/s3/credential_bridge.rs +++ b/native/core/src/cloud/s3/credential_bridge.rs @@ -185,6 +185,31 @@ pub enum AccessMode { Write = 1, } +/// A reference to a provider registered with the JVM dispatcher. `ensureInitialized` takes it, and +/// dropping the last bridge that shares it gives it back with `release`, so the dispatcher can +/// close a provider once nothing reads through it, such as after a catalog re-vends credentials +/// with a later expiry. +struct ProviderHandle(i64); + +impl Drop for ProviderHandle { + fn drop(&mut self) { + let released = JVMClasses::with_env(|env| -> Result<(), ExecutionError> { + unsafe { + jni_static_call!(env, + comet_s3_credential_dispatcher.release(self.0) -> () + )? + }; + Ok(()) + }); + if let Err(e) = released { + warn!( + "Failed to release S3 credential provider handle {}: {e}", + self.0 + ); + } + } +} + /// Credential provider that delegates to the JVM SPI via JNI. Instances live in the executor's /// FileIO and object store caches, so one serves many tasks. `handle` is the JVM-side identity /// for the `(provider_class, dispatch_key, catalog_properties)` triple returned by @@ -204,7 +229,7 @@ pub struct CometS3CredentialBridge { bucket: String, path: String, mode: AccessMode, - handle: i64, + handle: Arc, bucket_jstr: Arc>>, path_jstr: Arc>>, /// The credential this bridge last fetched, while it is fresh. A derived bridge starts with an @@ -217,7 +242,7 @@ impl fmt::Debug for CometS3CredentialBridge { f.debug_struct("CometS3CredentialBridge") .field("provider_class", &self.provider_class) .field("dispatch_key", &self.dispatch_key) - .field("handle", &self.handle) + .field("handle", &self.handle.0) .field("bucket", &self.bucket) .field("path", &self.path) .field("mode", &self.mode) @@ -264,7 +289,7 @@ impl CometS3CredentialBridge { bucket, path, mode, - handle, + handle: Arc::new(ProviderHandle(handle)), bucket_jstr, path_jstr, cache: CredentialCache::default(), @@ -290,7 +315,7 @@ impl CometS3CredentialBridge { bucket: self.bucket.clone(), path, mode: self.mode, - handle: self.handle, + handle: Arc::clone(&self.handle), bucket_jstr: Arc::clone(&self.bucket_jstr), path_jstr, cache: CredentialCache::default(), @@ -341,7 +366,7 @@ impl CometS3CredentialBridge { bucket, path, mode: self.mode, - handle: self.handle, + handle: Arc::clone(&self.handle), bucket_jstr, path_jstr, cache: CredentialCache::default(), @@ -355,7 +380,7 @@ impl CometS3CredentialBridge { let creds_obj: JObject = unsafe { jni_static_call!(env, comet_s3_credential_dispatcher.get_credentials_for_path( - self.handle, + self.handle.0, self.bucket_jstr.as_obj(), self.path_jstr.as_obj(), mode @@ -407,7 +432,7 @@ impl CometS3CredentialBridge { let locations: JObject = unsafe { jni_static_call!(env, comet_s3_credential_dispatcher.get_policy_locations( - self.handle, + self.handle.0, self.bucket_jstr.as_obj() ) -> JObject )? diff --git a/native/core/src/execution/operators/iceberg_common.rs b/native/core/src/execution/operators/iceberg_common.rs index f3f2286bb8a..2c76baa2722 100644 --- a/native/core/src/execution/operators/iceberg_common.rs +++ b/native/core/src/execution/operators/iceberg_common.rs @@ -153,11 +153,55 @@ impl FileIoCacheKey { properties: sorted_properties(catalog_properties), }) } + + /// Whether `other` is a `FileIO` for the same table, access mode and catalog, built from + /// credentials that this key's `FileIO` replaces. A catalog that re-vends credentials gives a + /// table new properties, and a commit moves its metadata file, so the table is compared by + /// [`Self::table_location`]. A commit that keeps the properties supersedes nothing, so + /// neighboring snapshots share one provider. + /// + /// Only a later `s3.session-token-expires-at-ms` supersedes, so both entries must carry one. + /// Properties can differ for reasons that say nothing about which is newer: two sessions or + /// tenants that share a catalog name can hold different tokens, OAuth credentials or tenant + /// ids for one table. Evicting on any difference would make them evict each other on every + /// task. Without a later expiry both entries stay, and the LRU bound retires the older one. + fn supersedes(&self, other: &Self) -> bool { + self.access_mode == other.access_mode + && self.catalog_name == other.catalog_name + && self.table_location() == other.table_location() + && self.properties != other.properties + && matches!( + (self.token_expires_at_ms(), other.token_expires_at_ms()), + (Some(new), Some(old)) if new > old + ) + } + + /// What identifies the table across commits. A read's reference path is a metadata file, and + /// Iceberg writes all of a table's metadata files into one directory (`write.metadata.path`, or + /// `/metadata`), so a read compares that directory. A write's reference path is the + /// table's data location, which a commit doesn't move. + fn table_location(&self) -> &str { + if self.access_mode == AccessMode::Read as u8 { + self.reference_path + .rfind('/') + .map_or(self.reference_path.as_str(), |i| &self.reference_path[..i]) + } else { + &self.reference_path + } + } + + fn token_expires_at_ms(&self) -> Option { + self.properties + .iter() + .find(|(k, _)| k == "s3.session-token-expires-at-ms") + .and_then(|(_, v)| v.parse().ok()) + } } const FILE_IO_CACHE_CAPACITY: usize = 64; -/// Least recently used entries are evicted first. +/// Least recently used entries are evicted first. Inserting a table's `FileIO` also evicts the ones +/// it supersedes, so their S3 credential providers can be released once no reader holds them. struct FileIoCache { entries: HashMap, order: VecDeque, @@ -182,17 +226,27 @@ impl FileIoCache { Some(file_io) } - /// Returns the replaced or evicted `FileIO` so the caller can drop it outside the lock. - fn insert(&mut self, key: FileIoCacheKey, file_io: FileIO) -> Option { + /// Returns the replaced or evicted `FileIO`s so the caller can drop them outside the lock. + fn insert(&mut self, key: FileIoCacheKey, file_io: FileIO) -> Vec { if let Some(previous) = self.entries.insert(key.clone(), file_io) { - return Some(previous); + return vec![previous]; } + let mut evicted = Vec::new(); + self.order.retain(|k| { + if key.supersedes(k) { + evicted.extend(self.entries.remove(k)); + false + } else { + true + } + }); self.order.push_back(key); if self.entries.len() > self.capacity { - let oldest = self.order.pop_front()?; - return self.entries.remove(&oldest); + if let Some(oldest) = self.order.pop_front() { + evicted.extend(self.entries.remove(&oldest)); + } } - None + evicted } } @@ -822,17 +876,236 @@ mod tests { ) .unwrap() }; - assert!(cache.insert(key("a"), local_file_io()).is_none()); - assert!(cache.insert(key("b"), local_file_io()).is_none()); + assert!(cache.insert(key("a"), local_file_io()).is_empty()); + assert!(cache.insert(key("b"), local_file_io()).is_empty()); assert!(cache.get(&key("a")).is_some()); - assert!(cache.insert(key("c"), local_file_io()).is_some()); + assert_eq!(cache.insert(key("c"), local_file_io()).len(), 1); assert!(cache.get(&key("b")).is_none()); assert!(cache.get(&key("a")).is_some()); assert!(cache.get(&key("c")).is_some()); - assert!(cache.insert(key("c"), local_file_io()).is_some()); + assert_eq!(cache.insert(key("c"), local_file_io()).len(), 1); + assert_eq!(cache.entries.len(), 2); + } + + /// Properties as a REST catalog vends them: a token and its expiry. `t1` expires before `t2`, + /// which expires before `t3`. + fn vended(token: &str) -> HashMap { + let n: u64 = token.trim_start_matches('t').parse().unwrap(); + HashMap::from([ + ("s3.session-token".to_string(), token.to_string()), + ( + "s3.session-token-expires-at-ms".to_string(), + (n * 1000).to_string(), + ), + ]) + } + + /// A catalog that re-vends credentials for a table gives it new properties with a later + /// expiry, so the table's new `FileIO` evicts the old one, and the old one's credential provider is released once its + /// readers finish. That holds whether the metadata file moved or not. Other tables, catalogs + /// and access modes keep theirs. + #[test] + fn cache_evicts_the_file_io_a_table_supersedes() { + let mut cache = FileIoCache::new(8); + let key = |token: &str, path: &str, catalog: &str, mode: AccessMode| { + FileIoCacheKey::new(&vended(token), path, catalog, mode).unwrap() + }; + let v1 = "s3://b/db/t/metadata/00001-a.metadata.json"; + let v2 = "s3://b/db/t/metadata/00002-b.metadata.json"; + let other_table = "s3://b/db/u/metadata/00001-c.metadata.json"; + assert!(cache + .insert(key("t1", v1, "cat", AccessMode::Read), local_file_io()) + .is_empty()); + assert!(cache + .insert( + key("t1", other_table, "cat", AccessMode::Read), + local_file_io() + ) + .is_empty()); + assert!(cache + .insert( + key("t1", v1, "other_cat", AccessMode::Read), + local_file_io() + ) + .is_empty()); + assert!(cache + .insert(key("t1", v1, "cat", AccessMode::Write), local_file_io()) + .is_empty()); + + // Re-vended credentials for the same metadata file. + assert_eq!( + cache + .insert(key("t2", v1, "cat", AccessMode::Read), local_file_io()) + .len(), + 1 + ); + assert!(cache.get(&key("t1", v1, "cat", AccessMode::Read)).is_none()); + + // A commit, with credentials vended alongside it. + assert_eq!( + cache + .insert(key("t3", v2, "cat", AccessMode::Read), local_file_io()) + .len(), + 1 + ); + assert!(cache.get(&key("t2", v1, "cat", AccessMode::Read)).is_none()); + assert!(cache.get(&key("t3", v2, "cat", AccessMode::Read)).is_some()); + + assert!(cache + .get(&key("t1", other_table, "cat", AccessMode::Read)) + .is_some()); + assert!(cache + .get(&key("t1", v1, "other_cat", AccessMode::Read)) + .is_some()); + assert!(cache + .get(&key("t1", v1, "cat", AccessMode::Write)) + .is_some()); + assert_eq!(cache.entries.len(), 4); + assert_eq!(cache.order.len(), 4); + } + + /// A commit that keeps the credentials leaves the previous snapshot's `FileIO` cached, so a + /// time-travel self-join or reads of neighboring snapshots don't rebuild it on each task. + /// With `write.metadata.path` set, a table's metadata files sit in that directory, which need + /// not end in `/metadata`. A commit under `s3://b/meta/db/t1` still supersedes the table's + /// previous `FileIO`, and two tables under `s3://b/metadata/db/` stay apart. + #[test] + fn cache_compares_reads_by_metadata_directory() { + let mut cache = FileIoCache::new(8); + let key = |token: &str, path: &str| { + FileIoCacheKey::new(&vended(token), path, "cat", AccessMode::Read).unwrap() + }; + let custom_v1 = "s3://b/meta/db/t1/00001-a.metadata.json"; + let custom_v2 = "s3://b/meta/db/t1/00002-b.metadata.json"; + assert!(cache + .insert(key("t1", custom_v1), local_file_io()) + .is_empty()); + assert_eq!(cache.insert(key("t2", custom_v2), local_file_io()).len(), 1); + assert!(cache.get(&key("t1", custom_v1)).is_none()); + + let shared_t1 = "s3://b/metadata/db/t1/00001-c.metadata.json"; + let shared_t2 = "s3://b/metadata/db/t2/00001-d.metadata.json"; + assert!(cache + .insert(key("t1", shared_t1), local_file_io()) + .is_empty()); + assert!(cache + .insert(key("t2", shared_t2), local_file_io()) + .is_empty()); + assert!(cache.get(&key("t1", shared_t1)).is_some()); + assert_eq!(cache.entries.len(), 3); + } + + #[test] + fn cache_keeps_the_file_io_of_a_commit_with_unchanged_properties() { + let mut cache = FileIoCache::new(8); + let properties = HashMap::from([("s3.session-token".to_string(), "t1".to_string())]); + let key = + |path: &str| FileIoCacheKey::new(&properties, path, "cat", AccessMode::Read).unwrap(); + let v1 = "s3://b/db/t/metadata/00001-a.metadata.json"; + let v2 = "s3://b/db/t/metadata/00002-b.metadata.json"; + assert!(cache.insert(key(v1), local_file_io()).is_empty()); + assert!(cache.insert(key(v2), local_file_io()).is_empty()); + assert!(cache.get(&key(v1)).is_some()); + assert!(cache.get(&key(v2)).is_some()); + } + + /// Two sessions reading one table with different vended tokens: only the token that expires + /// later evicts, so the sessions don't evict each other on every insert. + #[test] + fn cache_evicts_only_for_a_token_that_expires_later() { + let mut cache = FileIoCache::new(8); + let key = |token: &str, expires_at_ms: &str| { + FileIoCacheKey::new( + &HashMap::from([ + ("s3.session-token".to_string(), token.to_string()), + ( + "s3.session-token-expires-at-ms".to_string(), + expires_at_ms.to_string(), + ), + ]), + "s3://b/db/t/metadata/00001-a.metadata.json", + "cat", + AccessMode::Read, + ) + .unwrap() + }; + assert!(cache + .insert(key("later", "2000"), local_file_io()) + .is_empty()); + assert!(cache + .insert(key("earlier", "1000"), local_file_io()) + .is_empty()); + assert!(cache.get(&key("later", "2000")).is_some()); + assert!(cache.get(&key("earlier", "1000")).is_some()); + + assert_eq!( + cache.insert(key("latest", "3000"), local_file_io()).len(), + 2 + ); + assert!(cache.get(&key("latest", "3000")).is_some()); + assert_eq!(cache.entries.len(), 1); + } + + /// A write's reference path is the table's data location, compared whole: new properties for + /// it supersede, while a sibling table's location, which shares its parent, stays apart. + #[test] + fn cache_compares_writes_by_data_location() { + let mut cache = FileIoCache::new(8); + let key = |token: &str, path: &str| { + FileIoCacheKey::new(&vended(token), path, "cat", AccessMode::Write).unwrap() + }; + let t = "s3://b/warehouse/db/t"; + let u = "s3://b/warehouse/db/u"; + assert!(cache.insert(key("t1", t), local_file_io()).is_empty()); + assert!(cache.insert(key("t1", u), local_file_io()).is_empty()); + assert_eq!(cache.insert(key("t2", u), local_file_io()).len(), 1); + assert!(cache.get(&key("t1", t)).is_some()); + assert!(cache.get(&key("t1", u)).is_none()); assert_eq!(cache.entries.len(), 2); } + /// Properties that differ without a later expiry on both sides don't say which entry is newer, + /// for example two tenants sharing a catalog name with different OAuth tokens, or a token + /// vended without an expiry. Both entries stay, so neither side evicts the other on each task, + /// and the LRU bound retires the least recently used one. + #[test] + fn cache_keeps_entries_without_a_later_expiry() { + let mut cache = FileIoCache::new(3); + let key = |properties: &[(&str, &str)]| { + FileIoCacheKey::new( + &properties + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + "s3://b/db/t/metadata/00001-a.metadata.json", + "cat", + AccessMode::Read, + ) + .unwrap() + }; + let tenant_a = key(&[("token", "oauth-a"), ("header.X-Tenant", "a")]); + let tenant_b = key(&[("token", "oauth-b"), ("header.X-Tenant", "b")]); + let expiring = key(&[ + ("s3.session-token", "t1"), + ("s3.session-token-expires-at-ms", "1000"), + ]); + assert!(cache.insert(tenant_a.clone(), local_file_io()).is_empty()); + assert!(cache.insert(tenant_b.clone(), local_file_io()).is_empty()); + // An expiry on one side only doesn't order the two either way. + assert!(cache.insert(expiring.clone(), local_file_io()).is_empty()); + assert!(cache.get(&tenant_a).is_some()); + assert!(cache.get(&tenant_b).is_some()); + assert!(cache.get(&expiring).is_some()); + + // The LRU bound still applies: tenant_a is now the least recently used. + let unexpiring = key(&[("s3.session-token", "t2")]); + assert_eq!(cache.insert(unexpiring.clone(), local_file_io()).len(), 1); + assert!(cache.get(&tenant_a).is_none()); + assert!(cache.get(&tenant_b).is_some()); + assert!(cache.get(&unexpiring).is_some()); + assert_eq!(cache.entries.len(), 3); + } + fn factory_result(path: &str, mode: AccessMode) -> Result<(), String> { storage_factory_for(path, &HashMap::new(), "test_cat", mode) .map(|_| ()) diff --git a/native/core/src/lib.rs b/native/core/src/lib.rs index df6fb79e066..709f5b00e35 100644 --- a/native/core/src/lib.rs +++ b/native/core/src/lib.rs @@ -55,7 +55,7 @@ pub mod jvm_bridge { use errors::{try_unwrap_or_throw, CometError, CometResult}; use crate::cloud::s3::credential_bridge::AccessMode; -use crate::execution::operators::iceberg_common::builtin_storage_schemes; +use crate::execution::operators::iceberg_common::{builtin_storage_schemes, load_file_io}; pub mod alloc_accounting; pub mod cloud; @@ -321,6 +321,50 @@ pub extern "system" fn Java_org_apache_comet_NativeBase_icebergStorageSchemes( }) } +/// JNI, for tests: builds the Iceberg `FileIO` a read of `reference_path` with these catalog +/// properties uses, and keeps it in the `FileIO` cache as a read would. Building it registers the +/// configured S3 credential provider and makes no storage request, so a JVM test can check which +/// providers the cache holds without an S3 server. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeBase_loadIcebergFileIOForTesting( + env: EnvUnowned, + _: JClass, + reference_path: JString, + catalog_name: JString, + keys: JObjectArray, + values: JObjectArray, +) { + try_unwrap_or_throw(&env, |env| { + jvm_bridge::JVMClasses::init(env); + let reference_path: String = reference_path.try_to_string(env)?; + let catalog_name: String = catalog_name.try_to_string(env)?; + let mut properties = HashMap::new(); + for i in 0..keys.len(env)? { + let key = keys.get_element(env, i)?.try_to_string(env)?; + let value = values.get_element(env, i)?.try_to_string(env)?; + properties.insert(key, value); + } + load_file_io( + &properties, + &reference_path, + &catalog_name, + AccessMode::Read, + ) + .map_err(|e| CometError::Internal(e.to_string()))?; + Ok(()) + }) +} + +/// JNI, for tests: drops every cached Iceberg `FileIO`, as `release` does before it stops the +/// runtime, so the providers they hold are released. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeBase_clearIcebergFileIOCacheForTesting( + _env: EnvUnowned, + _: JClass, +) { + execution::operators::clear_file_io_cache(); +} + /// JNI: why the native scan cannot build its ABFS store for `url` from the object store options /// `keys`/`values`, or null when it can. The driver asks this when it plans a scan whose ABFS /// authentication resolved, so a store native would reject falls back to Spark instead of every diff --git a/native/jni-bridge/src/comet_s3_credential_dispatcher.rs b/native/jni-bridge/src/comet_s3_credential_dispatcher.rs index 210a8db89dc..880f42b3be1 100644 --- a/native/jni-bridge/src/comet_s3_credential_dispatcher.rs +++ b/native/jni-bridge/src/comet_s3_credential_dispatcher.rs @@ -38,6 +38,9 @@ pub struct CometS3CredentialDispatcher<'a> { /// implement `CometS3LocationScopedCredentialProvider`. pub method_get_policy_locations: JStaticMethodID, pub method_get_policy_locations_ret: ReturnType, + /// Gives back the provider reference `ensureInitialized` took. + pub method_release: JStaticMethodID, + pub method_release_ret: ReturnType, pub field_access_key_id: JFieldID, pub field_secret_access_key: JFieldID, pub field_session_token: JFieldID, @@ -73,6 +76,12 @@ impl<'a> CometS3CredentialDispatcher<'a> { jni::jni_sig!("(JLjava/lang/String;)[Ljava/lang/String;"), )?, method_get_policy_locations_ret: ReturnType::Array, + method_release: env.get_static_method_id( + JNIString::new(Self::JVM_CLASS), + jni::jni_str!("release"), + jni::jni_sig!("(J)V"), + )?, + method_release_ret: ReturnType::Primitive(Primitive::Void), field_access_key_id: env.get_field_id( &credentials_class, jni::jni_str!("accessKeyId"), diff --git a/spark/src/main/java/org/apache/comet/NativeBase.java b/spark/src/main/java/org/apache/comet/NativeBase.java index b9349057c99..1410a026c96 100644 --- a/spark/src/main/java/org/apache/comet/NativeBase.java +++ b/spark/src/main/java/org/apache/comet/NativeBase.java @@ -402,6 +402,22 @@ public static void releaseNative() throws Throwable { */ public static native String icebergStorageSchemes(boolean forWrite); + /** + * For tests: builds the Iceberg {@code FileIO} a read of {@code referencePath} with these catalog + * properties uses, and keeps it in the native {@code FileIO} cache as a read would. Building it + * registers the configured S3 credential provider and makes no storage request. + * + * @param referencePath the table metadata file the read would open + * @param catalogName the catalog name, which is the provider's dispatch key + * @param keys the catalog property keys + * @param values the catalog property values, in the order of {@code keys} + */ + public static native void loadIcebergFileIOForTesting( + String referencePath, String catalogName, String[] keys, String[] values); + + /** For tests: drops every cached Iceberg {@code FileIO}, releasing the providers they hold. */ + public static native void clearIcebergFileIOCacheForTesting(); + /** * Why the native scan cannot build its ABFS object store for {@code url} from the given object * store options, or null when it can. This runs the store construction the scan itself runs, so diff --git a/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcher.java b/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcher.java index 74015b4691b..575c6794401 100644 --- a/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcher.java +++ b/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcher.java @@ -26,6 +26,12 @@ import java.util.Map; import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; @@ -39,7 +45,20 @@ *

{@link #ensureInitialized} reflects the named class, runs {@code initialize(Map)} once, and * returns a {@code long} handle. {@link #getCredentialsForPath} takes that handle on every * per-request call. See the design notes in the contributor guide for why the SPI is shaped this - * way (keying, multi-tenant isolation, shutdown lifecycle). + * way (keying, multi-tenant isolation, lifecycle). + * + *

Each {@link #ensureInitialized} call takes a reference to the provider, and {@link #release} + * gives one back. When the last reference is released, the dispatcher closes the provider and + * forgets its key, so a later {@link #ensureInitialized} for the same key creates a new one. This + * is what retires a provider whose catalog properties were superseded, such as when an Iceberg REST + * catalog vends credentials with a later expiry for a table: native code releases the old handle + * when it drops the last reader built on it. Providers still referenced at JVM shutdown are closed + * then. + * + *

A release can come from any thread that drops a native reader, including a Tokio worker or a + * task thread inside a scan, so the dispatcher closes released providers on its own daemon thread. + * A {@code close()} that blocks, such as Iceberg's {@code VendedCredentialsProvider} waiting for a + * token refresh in flight, then delays only the closes queued behind it. */ public final class CometS3CredentialDispatcher { @@ -52,6 +71,15 @@ public final class CometS3CredentialDispatcher { private static final AtomicLong HANDLE_SEQ = new AtomicLong(1L); private static final CometS3AccessMode[] MODES = CometS3AccessMode.values(); + private static final ExecutorService DEFAULT_CLOSER = + Executors.newSingleThreadExecutor( + r -> { + Thread thread = new Thread(r, "comet-s3-credential-close"); + thread.setDaemon(true); + return thread; + }); + private static volatile Executor closer = DEFAULT_CLOSER; + static { Runtime.getRuntime() .addShutdownHook( @@ -63,7 +91,8 @@ private CometS3CredentialDispatcher() {} /** * Reflects and initializes the named provider on first call for the {@code (FQCN, dispatchKey, * catalogProperties)} triple, and returns a handle reused by subsequent {@link - * #getCredentialsForPath} calls. + * #getCredentialsForPath} calls. Each call takes a reference that the caller gives back with + * {@link #release}. */ public static long ensureInitialized( String providerClassName, String dispatchKey, Map catalogProperties) { @@ -77,15 +106,43 @@ public static long ensureInitialized( : Collections.unmodifiableMap(new HashMap<>(catalogProperties)); InstanceKey key = new InstanceKey(providerClassName, dispatchKey == null ? "" : dispatchKey, snapshot); - return KEY_TO_HANDLE.computeIfAbsent( - key, - k -> { - CometS3CredentialProvider provider = instantiate(k.providerClassName); - provider.initialize(k.catalogProperties); - long handle = HANDLE_SEQ.getAndIncrement(); - INSTANCES.put(handle, new RegisteredProvider(provider, k)); - return handle; - }); + while (true) { + long handle = + KEY_TO_HANDLE.computeIfAbsent( + key, + k -> { + CometS3CredentialProvider provider = instantiate(k.providerClassName); + provider.initialize(k.catalogProperties); + long h = HANDLE_SEQ.getAndIncrement(); + INSTANCES.put(h, new RegisteredProvider(provider, k)); + return h; + }); + RegisteredProvider registered = INSTANCES.get(handle); + if (registered != null && registered.acquire()) { + return handle; + } + // The provider was retired after the lookup; drop its stale mapping and register anew. + KEY_TO_HANDLE.remove(key, handle); + } + } + + /** + * Invoked by native code when it drops the last reader built on {@code handle}. Gives back one + * reference taken by {@link #ensureInitialized}, and closes the provider when it was the last. An + * unknown handle, such as one already closed at shutdown, is ignored. + */ + public static void release(long handle) { + RegisteredProvider registered = INSTANCES.get(handle); + if (registered == null || !registered.releaseLast()) { + return; + } + KEY_TO_HANDLE.remove(registered.key, handle); + INSTANCES.remove(handle, registered); + try { + closer.execute(() -> close(registered, "after its last reference was released")); + } catch (RejectedExecutionException e) { + close(registered, "after its last reference was released"); + } } /** @@ -198,20 +255,49 @@ private static CometS3CredentialProvider instantiate(String providerClassName) { /** Visible for tests; otherwise invoked from the JVM shutdown hook. */ static void closeAll() { for (RegisteredProvider registered : INSTANCES.values()) { - try { - registered.provider.close(); - } catch (Throwable t) { - LOG.warn( - "Exception from {} (dispatchKey={}).close() during shutdown", - registered.key.providerClassName, - registered.key.dispatchKey, - t); + // A concurrent last release may have retired the provider already, and closes it itself. + if (registered.retire()) { + close(registered, "during shutdown"); } } INSTANCES.clear(); KEY_TO_HANDLE.clear(); } + /** + * Visible for tests: runs closes of released providers on {@code executor}, or on the + * dispatcher's own thread when it is null. + */ + static void setCloserForTesting(Executor executor) { + closer = executor == null ? DEFAULT_CLOSER : executor; + } + + /** + * Visible for tests: waits until the dispatcher's own thread has run every close queued before + * this call. Its executor runs one task at a time, in order, so a no-op queued now runs last. + */ + static void awaitQueuedClosesForTesting() throws Exception { + DEFAULT_CLOSER.submit(() -> {}).get(10, TimeUnit.SECONDS); + } + + /** Visible for tests: the number of providers the dispatcher holds. */ + static int registeredCount() { + return INSTANCES.size(); + } + + private static void close(RegisteredProvider registered, String when) { + try { + registered.provider.close(); + } catch (Throwable t) { + LOG.warn( + "Exception from {} (dispatchKey={}).close() {}", + registered.key.providerClassName, + registered.key.dispatchKey, + when, + t); + } + } + private static final class InstanceKey { final String providerClassName; final String dispatchKey; @@ -241,12 +327,50 @@ public int hashCode() { } private static final class RegisteredProvider { + /** {@link #refs} once the provider is retired; it then takes no more references. */ + private static final int RETIRED = -1; + final CometS3CredentialProvider provider; final InstanceKey key; + private final AtomicInteger refs = new AtomicInteger(); RegisteredProvider(CometS3CredentialProvider provider, InstanceKey key) { this.provider = provider; this.key = key; } + + /** Takes a reference, unless the provider is already retired. */ + boolean acquire() { + while (true) { + int n = refs.get(); + if (n == RETIRED) { + return false; + } + if (refs.compareAndSet(n, n + 1)) { + return true; + } + } + } + + /** Retires the provider outright. Returns false when it was already retired. */ + boolean retire() { + return refs.getAndSet(RETIRED) != RETIRED; + } + + /** + * Gives back a reference. Returns true when it was the last, in which case the provider is now + * retired and the caller closes it. A concurrent {@link #acquire} that wins the race keeps it. + */ + boolean releaseLast() { + while (true) { + int n = refs.get(); + if (n <= 0) { + return false; + } + if (n == 1 ? refs.compareAndSet(1, RETIRED) : refs.compareAndSet(n, n - 1)) { + return n == 1; + } + } + } } } diff --git a/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialProvider.java b/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialProvider.java index 2476c39dd9a..63eab9ad76d 100644 --- a/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialProvider.java +++ b/spark/src/main/java/org/apache/comet/cloud/s3/CometS3CredentialProvider.java @@ -50,8 +50,15 @@ default void initialize(Map catalogProperties) {} CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) throws Exception; /** - * Invoked from the dispatcher's best-effort JVM shutdown hook. Default no-op suits stateless + * Invoked once nothing reads through this instance any more, such as after an Iceberg catalog + * re-vends credentials for a table with a later {@code s3.session-token-expires-at-ms} or the + * executor's {@code FileIO} cache evicts the entry built on it, and from the dispatcher's + * best-effort JVM shutdown hook for instances still in use then. Default no-op suits stateless * providers; override to release HTTP clients, refresh executors, etc. + * + *

A release while the job runs closes the instance on a dispatcher thread, possibly while a + * new instance for the same key initializes, so release only this instance's own resources, not + * state other instances share. Comet calls it at most once. */ @Override default void close() throws Exception {} diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 73f19b2cfb2..0d90092a7e2 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -770,11 +770,11 @@ case class CometScanRule(session: SparkSession) fileIOProperties.contains( s"hadoop.dfs.ha.namenodes.${key.stripPrefix("hdfs.name-node.")}") } - val catalogProperties = + val catalogProperties = CometIcebergNativeScan.withVendedCredentialProvider( hadoopDerivedProperties ++ derivedHdfsProperties ++ fileIOProperties ++ hadoopS3Options .get(COMET_S3_COMPLIANT_SCHEMES_KEY) - .map(COMET_S3_COMPLIANT_SCHEMES_KEY -> _) + .map(COMET_S3_COMPLIANT_SCHEMES_KEY -> _)) val result = CometIcebergNativeScanMetadata.extract( scanExec.scan, diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala index 8d08e06d334..fbd564643c0 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeScan.scala @@ -50,6 +50,7 @@ import org.apache.comet.objectstore.NativeConfig import org.apache.comet.serde.{CometOperatorSerde, OperatorOuterClass} import org.apache.comet.serde.OperatorOuterClass.{Operator, SparkStructField} import org.apache.comet.serde.QueryPlanSerde.serializeDataType +import org.apache.comet.util.ClassLoaders object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] with Logging { @@ -607,6 +608,94 @@ object CometIcebergNativeScan extends CometOperatorSerde[CometBatchScanExec] wit } } + /** Activation key for the `CometS3CredentialProvider` SPI on the native Iceberg path. */ + private[comet] val IcebergCredentialProviderClassKey = "s3.comet.credential.provider.class" + + /** The provider that refreshes REST-vended credentials (Spark 4.x builds only). */ + private[comet] val VendedCredentialProviderClass = + "org.apache.comet.cloud.s3.IcebergVendedS3CredentialProvider" + + // Iceberg's AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT and REFRESH_CREDENTIALS_ENABLED. + private val RefreshCredentialsEndpointKey = "client.refresh-credentials-endpoint" + private val RefreshCredentialsEnabledKey = "client.refresh-credentials-enabled" + + // Iceberg's CatalogProperties.URI, which a relative refresh endpoint resolves against. + private val CatalogUriKey = "uri" + + /** + * Whether the provider can initialize in this JVM. It ships in Comet's jar, so it and the AWS + * SDK interface it implements come from Comet's loader. It builds its delegate through + * `AwsClientProperties` from the context loader and casts the result to that interface, so both + * loaders must see one copy of the SDK. A provider that fails to initialize falls back to the + * default chain on every task, so the check runs once here instead. + */ + private lazy val vendedCredentialProviderAvailable: Boolean = { + val cometLoader = getClass.getClassLoader + val usable = + vendedCredentialProviderUsable(cometLoader, ClassLoaders.contextOrDefault(cometLoader)) + if (!usable) { + logInfo( + s"$VendedCredentialProviderClass is not usable in this JVM, so native Iceberg scans " + + "keep the credentials vended at table load and are not refreshed") + } + usable + } + + /** + * The check behind [[vendedCredentialProviderAvailable]], for the loaders Comet and Iceberg + * come from. It fails when Comet's loader can't see the AWS SDK, such as when the only copy is + * on Spark's user-jar loader, and when `AwsClientProperties` links against another copy of it. + */ + private[comet] def vendedCredentialProviderUsable( + cometLoader: ClassLoader, + contextLoader: ClassLoader): Boolean = + // scalastyle:off classforname + try { + Class.forName(VendedCredentialProviderClass, false, cometLoader) + val sdkProvider = Class.forName(AwsCredentialsProviderClass, false, cometLoader) + val clientProperties = Class.forName(AwsClientPropertiesClass, false, contextLoader) + Class.forName(AwsCredentialsProviderClass, false, clientProperties.getClassLoader) eq + sdkProvider + } catch { + // A LinkageError, such as part of the SDK missing, means the same as a missing class here. + case _: ClassNotFoundException | _: LinkageError => false + } + // scalastyle:on classforname + + private val AwsCredentialsProviderClass = + "software.amazon.awssdk.auth.credentials.AwsCredentialsProvider" + private val AwsClientPropertiesClass = "org.apache.iceberg.aws.AwsClientProperties" + + /** + * Names [[VendedCredentialProviderClass]] as the credential provider when the catalog returned + * a refresh endpoint and the user named no provider. + * + * Without a provider, the native FileIO signs with the `s3.*` credentials vended when the + * driver loaded the table, and a scan that outlives them fails with `ExpiredToken`. Iceberg's + * `S3FileIO` refreshes them through the endpoint instead. An explicit provider key, including + * an empty one, is left alone so it stays an opt-out. Builds without the provider class (Spark + * 3.x) keep the vended credentials as they are, and so does a catalog with no `uri`: Iceberg's + * `VendedCredentialsProvider` needs it to reach the catalog whether the endpoint is relative or + * absolute. + */ + def withVendedCredentialProvider(properties: Map[String, String]): Map[String, String] = + withVendedCredentialProvider(properties, vendedCredentialProviderAvailable) + + private[comet] def withVendedCredentialProvider( + properties: Map[String, String], + providerAvailable: Boolean): Map[String, String] = { + val hasEndpoint = properties.get(RefreshCredentialsEndpointKey).exists(_.trim.nonEmpty) + val hasCatalogUri = properties.get(CatalogUriKey).exists(_.trim.nonEmpty) + val vendsRefreshEndpoint = hasEndpoint && hasCatalogUri && + !properties.get(RefreshCredentialsEnabledKey).exists(_.trim.equalsIgnoreCase("false")) + if (vendsRefreshEndpoint && providerAvailable && + !properties.contains(IcebergCredentialProviderClassKey)) { + properties + (IcebergCredentialProviderClassKey -> VendedCredentialProviderClass) + } else { + properties + } + } + /** * Transforms Hadoop S3A configuration keys to Iceberg FileIO property keys. * diff --git a/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProvider.java b/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProvider.java new file mode 100644 index 00000000000..afb8171c04d --- /dev/null +++ b/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProvider.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.cloud.s3; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; + +import org.apache.comet.util.ClassLoaders; + +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +/** + * Resolves S3 credentials for native Iceberg scans the way Iceberg's own {@code S3FileIO} does, so + * credentials vended by a REST catalog are refreshed instead of expiring mid-job. + * + *

Without a provider, the native FileIO signs every request with the {@code s3.*} credentials + * the catalog vended when the driver loaded the table. Vended credentials are short-lived, so a + * scan still running after they expire fails with {@code ExpiredToken}, while Spark's own read of + * the same table refreshes them through {@code client.refresh-credentials-endpoint}. + * + *

This provider hands the FileIO property bag to Iceberg's {@code AwsClientProperties}, which + * picks the same credentials provider {@code S3FileIO} would: a {@code VendedCredentialsProvider} + * when the catalog returned a refresh endpoint, the static vended keys otherwise, and the AWS + * default chain when neither is set. Iceberg is loaded by reflection from the context class loader, + * as elsewhere in Comet, so Comet keeps no compile-time Iceberg dependency. + * + *

Comet selects this provider automatically when the FileIO properties carry a refresh endpoint + * and no provider is configured (see {@code CometIcebergNativeScan.withVendedCredentialProvider}). + * To name it explicitly: {@code spark.sql.catalog..s3.comet.credential.provider.class = + * org.apache.comet.cloud.s3.IcebergVendedS3CredentialProvider}. + */ +public final class IcebergVendedS3CredentialProvider implements CometS3CredentialProvider { + + static final String AWS_CLIENT_PROPERTIES_CLASS = "org.apache.iceberg.aws.AwsClientProperties"; + + // S3FileIOProperties.ACCESS_KEY_ID, SECRET_ACCESS_KEY, SESSION_TOKEN and + // SESSION_TOKEN_EXPIRES_AT_MS. + static final String ACCESS_KEY_ID = "s3.access-key-id"; + static final String SECRET_ACCESS_KEY = "s3.secret-access-key"; + static final String SESSION_TOKEN = "s3.session-token"; + static final String SESSION_TOKEN_EXPIRES_AT_MS = "s3.session-token-expires-at-ms"; + + private static final String ANONYMOUS_HINT = + "Iceberg resolved this credentials provider from the catalog's FileIO properties (for" + + " example client.credentials-provider). To read the table without Comet's credential" + + " provider, set spark.sql.catalog..s3.comet.credential.provider.class to an" + + " empty value."; + + private volatile AwsCredentialsProvider delegate; + private volatile String vendedAccessKeyId; + private volatile long vendedExpirationEpochMillis; + + @Override + public void initialize(Map catalogProperties) { + ClassLoader loader = ClassLoaders.contextOrDefault(getClass().getClassLoader()); + Class clientPropertiesClass; + try { + clientPropertiesClass = Class.forName(AWS_CLIENT_PROPERTIES_CLASS, true, loader); + } catch (ClassNotFoundException | LinkageError e) { + throw new IllegalStateException( + "IcebergVendedS3CredentialProvider needs iceberg-aws (" + + AWS_CLIENT_PROPERTIES_CLASS + + ") on the application class path", + e); + } + Object provider; + try { + Object clientProperties = + clientPropertiesClass.getConstructor(Map.class).newInstance(catalogProperties); + Method credentialsProvider = + clientPropertiesClass.getMethod( + "credentialsProvider", String.class, String.class, String.class); + provider = + credentialsProvider.invoke( + clientProperties, + catalogProperties.get(ACCESS_KEY_ID), + catalogProperties.get(SECRET_ACCESS_KEY), + catalogProperties.get(SESSION_TOKEN)); + } catch (InvocationTargetException e) { + throw new IllegalStateException( + "Iceberg AwsClientProperties failed to build a credentials provider", e.getCause()); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException( + "This iceberg-aws version has no AwsClientProperties.credentialsProvider(String, String," + + " String)", + e); + } + this.delegate = asSdkProvider(provider); + this.vendedAccessKeyId = catalogProperties.get(ACCESS_KEY_ID); + this.vendedExpirationEpochMillis = parseEpochMillis(catalogProperties); + } + + /** + * Casts Iceberg's provider to the AWS SDK interface Comet links against. The two only match when + * the SDK is loaded once, by a loader both Iceberg and Comet see. + */ + private static AwsCredentialsProvider asSdkProvider(Object provider) { + try { + return (AwsCredentialsProvider) provider; + } catch (NoClassDefFoundError e) { + throw new IllegalStateException( + "IcebergVendedS3CredentialProvider needs the AWS SDK v2 on Comet's class path, next to" + + " the Comet jar", + e); + } catch (ClassCastException e) { + throw new IllegalStateException( + "Iceberg returned a credentials provider from a different copy of the AWS SDK than the" + + " one Comet loaded. This happens when the SDK is on both the application and the" + + " Spark class paths with spark.{driver,executor}.userClassPathFirst=true; keep one" + + " copy of the SDK.", + e); + } + } + + private static long parseEpochMillis(Map catalogProperties) { + String value = catalogProperties.get(SESSION_TOKEN_EXPIRES_AT_MS); + if (value == null || value.trim().isEmpty()) { + return 0L; + } + try { + return Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + return 0L; + } + } + + @Override + public CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) { + AwsCredentialsProvider p = delegate; + if (p == null) { + throw new IllegalStateException( + "IcebergVendedS3CredentialProvider used before initialize(Map) was called; " + + "Comet should always invoke initialize before getCredentialsForPath"); + } + AwsCredentials c = p.resolveCredentials(); + CometS3Credentials credentials = + SdkCredentialExtraction.toCometCredentials(context.getBucket(), c, ANONYMOUS_HINT); + // Static vended keys carry no expiry in the SDK object, so report the one the catalog vended + // with them. Comet then keeps reusing the credential until shortly before it expires. + if (credentials.getExpirationEpochMillis() == 0L + && vendedExpirationEpochMillis > 0L + && c.accessKeyId().equals(vendedAccessKeyId)) { + return new CometS3Credentials( + credentials.getAccessKeyId(), + credentials.getSecretAccessKey(), + credentials.getSessionToken(), + vendedExpirationEpochMillis); + } + return credentials; + } + + @Override + public void close() throws Exception { + AwsCredentialsProvider p = delegate; + if (p instanceof AutoCloseable) { + ((AutoCloseable) p).close(); + } + } +} diff --git a/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java b/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java index e96b037453f..e1b778b620b 100644 --- a/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java +++ b/spark/src/main/spark-4.x/org/apache/comet/cloud/s3/SdkCredentialExtraction.java @@ -34,6 +34,21 @@ final class SdkCredentialExtraction { private SdkCredentialExtraction() {} static CometS3Credentials toCometCredentials(String bucket, AwsCredentials creds) { + return toCometCredentials( + bucket, + creds, + "The Comet S3 credential adapters cannot serve anonymous access; opt this bucket out with" + + " an empty fs.s3a.bucket." + + bucket + + ".comet.credential.provider.class so the native reader accesses it unsigned."); + } + + /** + * As {@link #toCometCredentials(String, AwsCredentials)}, with {@code anonymousHint} telling the + * user how to opt out when the provider resolved anonymous or empty credentials. + */ + static CometS3Credentials toCometCredentials( + String bucket, AwsCredentials creds, String anonymousHint) { String accessKeyId = creds.accessKeyId(); String secretAccessKey = creds.secretAccessKey(); if (accessKeyId == null @@ -43,15 +58,12 @@ static CometS3Credentials toCometCredentials(String bucket, AwsCredentials creds // The chain resolved anonymous or empty credentials (e.g. AnonymousCredentialsProvider, used // for public buckets, or a provider that returns blank keys). The SPI has no way to express // "anonymous", and the native reader would otherwise sign with blank keys and 403, so fail - // with a clear cause instead. To read this bucket anonymously, opt it out with an empty - // fs.s3a.bucket..comet.credential.provider.class. + // with a clear cause instead. throw new IllegalStateException( "The credential provider chain resolved anonymous or empty credentials for bucket " + bucket - + ". The Comet S3 credential adapters cannot serve anonymous access; opt this bucket" - + " out with an empty fs.s3a.bucket." - + bucket - + ".comet.credential.provider.class so the native reader accesses it unsigned."); + + ". " + + anonymousHint); } String sessionToken = null; long expirationEpochMillis = 0L; diff --git a/spark/src/test/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcherTest.java b/spark/src/test/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcherTest.java index 9601497f1ba..0ab4d89403b 100644 --- a/spark/src/test/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcherTest.java +++ b/spark/src/test/java/org/apache/comet/cloud/s3/CometS3CredentialDispatcherTest.java @@ -19,10 +19,18 @@ package org.apache.comet.cloud.s3; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -42,11 +50,21 @@ public class CometS3CredentialDispatcherTest { private static final int WRITE = CometS3AccessMode.WRITE.ordinal(); @Before - public void resetTestProvider() { + public void resetTestProvider() throws Exception { // Each test starts with an empty dispatcher cache so providers and counters from prior tests - // do not leak through closeAll() / initCount / closeCount assertions. + // do not leak through closeAll() / initCount / closeCount assertions. A close still queued on + // the dispatcher's thread would count against this test, so those finish first. + CometS3CredentialDispatcher.awaitQueuedClosesForTesting(); CometS3CredentialDispatcher.closeAll(); TestCometS3CredentialProvider.reset(); + // Close released providers on the releasing thread, so counts are checked right after release. + CometS3CredentialDispatcher.setCloserForTesting(Runnable::run); + } + + @After + public void restoreCloser() throws Exception { + CometS3CredentialDispatcher.setCloserForTesting(null); + CometS3CredentialDispatcher.awaitQueuedClosesForTesting(); } private long init() { @@ -267,4 +285,151 @@ public void closeAllSwallowsProviderExceptions() { // Both close() invocations ran even though one (the first to be visited) threw. assertEquals(2, TestCometS3CredentialProvider.closeCount.get()); } + + @Test + public void releasingTheLastReferenceClosesTheProvider() throws Exception { + long h1 = init(); + long h2 = init(); + assertEquals(h1, h2); + + CometS3CredentialDispatcher.release(h1); + assertEquals(0, TestCometS3CredentialProvider.closeCount.get()); + CometS3CredentialDispatcher.getCredentialsForPath(h2, "b", "k", READ); + + CometS3CredentialDispatcher.release(h2); + assertEquals(1, TestCometS3CredentialProvider.closeCount.get()); + assertEquals(0, CometS3CredentialDispatcher.registeredCount()); + Exception thrown = + assertThrows( + IllegalStateException.class, + () -> CometS3CredentialDispatcher.getCredentialsForPath(h2, "b", "k", READ)); + assertTrue(thrown.getMessage().contains("not initialized")); + } + + @Test + public void aRetiredKeyGetsANewProvider() { + long retired = init(); + CometS3CredentialDispatcher.release(retired); + + long fresh = init(); + + assertNotEquals(retired, fresh); + assertEquals(2, TestCometS3CredentialProvider.initCount.get()); + assertEquals(1, CometS3CredentialDispatcher.registeredCount()); + } + + /** An extra release, or one for a handle closed at shutdown, changes nothing. */ + @Test + public void unknownOrRepeatedReleasesAreIgnored() { + CometS3CredentialDispatcher.release(Long.MAX_VALUE); + long handle = init(); + CometS3CredentialDispatcher.release(handle); + CometS3CredentialDispatcher.release(handle); + + assertEquals(1, TestCometS3CredentialProvider.closeCount.get()); + long other = init(); + CometS3CredentialDispatcher.release(handle); + assertEquals(1, CometS3CredentialDispatcher.registeredCount()); + CometS3CredentialDispatcher.release(other); + assertEquals(2, TestCometS3CredentialProvider.closeCount.get()); + } + + @Test + public void concurrentAcquireAndReleaseNeverUseAClosedProvider() throws Exception { + int threads = 8; + int rounds = 2000; + ExecutorService pool = Executors.newFixedThreadPool(threads); + try { + List> futures = new ArrayList<>(); + for (int t = 0; t < threads; t++) { + futures.add( + pool.submit( + () -> { + for (int i = 0; i < rounds; i++) { + long handle = init(); + CometS3CredentialDispatcher.getCredentialsForPath(handle, "b", "k", READ); + CometS3CredentialDispatcher.release(handle); + } + return null; + })); + } + for (Future f : futures) { + f.get(); + } + } finally { + pool.shutdownNow(); + } + assertEquals(0, CometS3CredentialDispatcher.registeredCount()); + assertEquals( + TestCometS3CredentialProvider.initCount.get(), + TestCometS3CredentialProvider.closeCount.get()); + } + + /** The releasing thread may be a native reader's, so the close runs on the dispatcher's own. */ + @Test + public void releaseClosesOnTheDispatcherThread() throws Exception { + CometS3CredentialDispatcher.setCloserForTesting(null); + TestCometS3CredentialProvider.closeFinished = new CountDownLatch(1); + long handle = init(); + + CometS3CredentialDispatcher.release(handle); + + // Waiting for the whole close keeps its count out of the next test's. + assertTrue(TestCometS3CredentialProvider.closeFinished.await(10, TimeUnit.SECONDS)); + assertEquals(1, TestCometS3CredentialProvider.closeCount.get()); + assertEquals("comet-s3-credential-close", TestCometS3CredentialProvider.lastCloseThread); + assertEquals(0, CometS3CredentialDispatcher.registeredCount()); + } + + /** A close that blocks holds up neither the releasing thread nor a new reader for the key. */ + @Test + public void aBlockingCloseDoesNotHoldUpRelease() throws Exception { + CometS3CredentialDispatcher.setCloserForTesting(null); + CountDownLatch block = new CountDownLatch(1); + TestCometS3CredentialProvider.blockClose = block; + TestCometS3CredentialProvider.closeEntered = new CountDownLatch(1); + CountDownLatch finished = new CountDownLatch(1); + TestCometS3CredentialProvider.closeFinished = finished; + try { + long retired = init(); + CometS3CredentialDispatcher.release(retired); + assertTrue(TestCometS3CredentialProvider.closeEntered.await(10, TimeUnit.SECONDS)); + + long fresh = init(); + CometS3CredentialDispatcher.getCredentialsForPath(fresh, "b", "k", READ); + assertNotEquals(retired, fresh); + assertEquals(0, TestCometS3CredentialProvider.closeCount.get()); + } finally { + block.countDown(); + } + assertTrue(finished.await(10, TimeUnit.SECONDS)); + assertEquals(1, TestCometS3CredentialProvider.closeCount.get()); + } + + /** + * The shutdown hook and a concurrent last release can reach one provider together. Only one of + * them closes it. + */ + @Test + public void shutdownAndAConcurrentLastReleaseCloseOnce() throws Exception { + long handle = init(); + CountDownLatch block = new CountDownLatch(1); + TestCometS3CredentialProvider.blockClose = block; + TestCometS3CredentialProvider.closeEntered = new CountDownLatch(1); + Thread shutdown = new Thread(CometS3CredentialDispatcher::closeAll); + Thread releaser = new Thread(() -> CometS3CredentialDispatcher.release(handle)); + shutdown.start(); + try { + assertTrue(TestCometS3CredentialProvider.closeEntered.await(10, TimeUnit.SECONDS)); + // Without the shared claim, this release would close the provider a second time. + releaser.start(); + releaser.join(2_000); + } finally { + block.countDown(); + } + shutdown.join(10_000); + releaser.join(10_000); + + assertEquals(1, TestCometS3CredentialProvider.closeCount.get()); + } } diff --git a/spark/src/test/java/org/apache/comet/cloud/s3/TestCometS3CredentialProvider.java b/spark/src/test/java/org/apache/comet/cloud/s3/TestCometS3CredentialProvider.java index 77ce86c8369..0286b3ac589 100644 --- a/spark/src/test/java/org/apache/comet/cloud/s3/TestCometS3CredentialProvider.java +++ b/spark/src/test/java/org/apache/comet/cloud/s3/TestCometS3CredentialProvider.java @@ -20,11 +20,13 @@ package org.apache.comet.cloud.s3; import java.util.Map; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicInteger; /** * Test-only provider instantiated by the dispatcher when its FQCN is passed across JNI. State is - * static because the dispatcher caches one instance per (FQCN, dispatchKey) for the JVM lifetime. + * static because the dispatcher keeps one instance per (FQCN, dispatchKey, catalogProperties) until + * its last reference is released or the JVM shuts down. */ public class TestCometS3CredentialProvider implements CometS3CredentialProvider { @@ -40,7 +42,18 @@ public class TestCometS3CredentialProvider implements CometS3CredentialProvider static volatile CometS3Credentials nextResult = new CometS3Credentials("AKIATEST", "secret", "session-tok", 0L); + /** When set, {@code close()} counts down {@link #closeEntered} and waits for this latch. */ + static volatile CountDownLatch blockClose; + + static volatile CountDownLatch closeEntered; + + /** When set, {@code close()} counts this down once it has counted the close. */ + static volatile CountDownLatch closeFinished; + + static volatile String lastCloseThread; + private volatile String tenantId; + private volatile boolean closed; static void reset() { initCount.set(0); @@ -53,6 +66,10 @@ static void reset() { throwOnNext = null; throwOnClose = null; nextResult = new CometS3Credentials("AKIATEST", "secret", "session-tok", 0L); + blockClose = null; + closeEntered = null; + closeFinished = null; + lastCloseThread = null; } @Override @@ -63,6 +80,9 @@ public void initialize(Map catalogProperties) { @Override public CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) { + if (closed) { + throw new IllegalStateException("TestCometS3CredentialProvider used after close()"); + } callCount.incrementAndGet(); lastBucket = context.getBucket(); lastPath = context.getPath(); @@ -78,11 +98,28 @@ public CometS3Credentials getCredentialsForPath(CometS3CredentialContext context @Override public void close() throws Exception { - closeCount.incrementAndGet(); - Exception toThrow = throwOnClose; - if (toThrow != null) { - throwOnClose = null; - throw toThrow; + closed = true; + lastCloseThread = Thread.currentThread().getName(); + CountDownLatch entered = closeEntered; + if (entered != null) { + entered.countDown(); + } + try { + CountDownLatch block = blockClose; + if (block != null) { + block.await(); + } + closeCount.incrementAndGet(); + Exception toThrow = throwOnClose; + if (toThrow != null) { + throwOnClose = null; + throw toThrow; + } + } finally { + CountDownLatch finished = closeFinished; + if (finished != null) { + finished.countDown(); + } } } } diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 85fd3ad3f43..dd86fa2c273 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -54,6 +54,7 @@ import org.apache.comet.cloud.s3.MinioLocationScopedCredentialProvider import org.apache.comet.iceberg.{IcebergReflection, RESTCatalogHelper} import org.apache.comet.rules.CometScanRule import org.apache.comet.serde.{OperatorOuterClass, QueryPlanSerde} +import org.apache.comet.serde.operator.CometIcebergNativeScan import org.apache.comet.testing.{FuzzDataGenerator, SchemaGenOptions} /** @@ -5184,6 +5185,48 @@ class CometIcebergNativeSuite } } + test("REST catalog refresh endpoint selects the vended provider for the native scan") { + assume(!isSpark42Plus, "https://github.com/apache/datafusion-comet/issues/4969") + assume(icebergAvailable, "Iceberg not available in classpath") + + // A local warehouse never builds an S3 bridge, so the scan runs without calling the endpoint. + // Catalogs usually return a relative endpoint, which the provider resolves against `uri`. + val endpoint = Map("client.refresh-credentials-endpoint" -> "/v1/credentials") + withRESTCatalog(vendedCredentials = endpoint) { (restUri, _, warehouseDir) => + withSQLConf( + "spark.sql.catalog.rest_vend" -> "org.apache.iceberg.spark.SparkCatalog", + "spark.sql.catalog.rest_vend.catalog-impl" -> "org.apache.iceberg.rest.RESTCatalog", + "spark.sql.catalog.rest_vend.uri" -> restUri, + "spark.sql.catalog.rest_vend.warehouse" -> warehouseDir.getAbsolutePath, + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { + + spark.sql("CREATE NAMESPACE rest_vend.db") + spark.sql("CREATE TABLE rest_vend.db.t (id INT) USING iceberg") + spark.sql("INSERT INTO rest_vend.db.t VALUES (1), (2), (3)") + + val (_, plan) = checkSparkAnswer("SELECT * FROM rest_vend.db.t ORDER BY id") + val scans = collectIcebergNativeScans(plan) + assert(scans.nonEmpty, s"Expected a CometIcebergNativeScanExec in:\n$plan") + // The provider ships only in Spark 4.x builds. + val expected = + if (isSpark40Plus) Some(CometIcebergNativeScan.VendedCredentialProviderClass) else None + scans.foreach { scan => + val properties = scan.nativeIcebergScanMetadata.catalogProperties + val selected = properties.get(CometIcebergNativeScan.IcebergCredentialProviderClassKey) + assert(selected == expected, s"Serialized provider $selected, expected $expected") + assert( + properties.get("uri").contains(restUri), + s"Serialized uri ${properties.get("uri")}, expected $restUri") + } + + spark.sql("DROP TABLE rest_vend.db.t") + spark.sql("DROP NAMESPACE rest_vend.db") + } + } + } + test("runtime filtering - multiple DPP filters on two partition columns") { assume(icebergAvailable, "Iceberg not available") withTempIcebergDir { warehouseDir => diff --git a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala index f3bc317c295..4afc293c17d 100644 --- a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialBridgeSuite.scala @@ -256,6 +256,46 @@ class CometS3CredentialBridgeSuite } } + test("a table's superseded FileIO releases its CometS3CredentialProvider") { + assume(icebergAvailable, "Iceberg not available in classpath") + + spark.sql("CREATE TABLE s3_catalog.db.bridge_release (id INT) USING iceberg") + spark.sql("INSERT INTO s3_catalog.db.bridge_release VALUES (1), (2), (3)") + + // Each session builds its own catalog, so a different instance tag and a later expiry give + // the same table the FileIO properties of a catalog that re-vends credentials. Only a later + // expiry supersedes, so each read's expiry is later than the one before. + val firstExpiry = System.currentTimeMillis() + 3600000L + def readWithTag(tag: String, expiresAtMs: Long): Unit = { + val session = spark.newSession() + session.conf.set( + s"spark.sql.catalog.s3_catalog.${MinioCometS3CredentialProvider.TEST_INSTANCE_TAG}", + tag) + session.conf.set( + "spark.sql.catalog.s3_catalog.s3.session-token-expires-at-ms", + expiresAtMs.toString) + val df = session.sql("SELECT * FROM s3_catalog.db.bridge_release") + assertHasCometIcebergScan(df.queryExecution.executedPlan) + // collect, not count: Iceberg can answer a count from metadata without reading files. + assert(df.collect().length == 3) + } + + readWithTag("release-a", firstExpiry) + val afterFirst = CometS3CredentialDispatcher.registeredCount() + assert(MinioCometS3CredentialProvider.initPropertiesForTag("release-a") != null) + + // The second read registers a provider for its own FileIO, and evicting the first FileIO + // drops the last bridge holding the first provider, which releases it. + readWithTag("release-b", firstExpiry + 3600000L) + assert(MinioCometS3CredentialProvider.initPropertiesForTag("release-b") != null) + assert( + CometS3CredentialDispatcher.registeredCount() == afterFirst, + s"Expected the superseded provider to be released: ${CometS3CredentialDispatcher + .registeredCount()} registered after the second read, $afterFirst after the first") + + spark.sql("DROP TABLE s3_catalog.db.bridge_release") + } + // Minio does not enforce per-prefix policies here, so these tests check the path of each // credential request: a read must request the credential of its longest covering location. diff --git a/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialReleaseSuite.scala b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialReleaseSuite.scala new file mode 100644 index 00000000000..18383397bfd --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/cloud/s3/CometS3CredentialReleaseSuite.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.cloud.s3 + +import org.apache.spark.sql.CometTestBase + +import org.apache.comet.NativeBase + +/** + * Builds native Iceberg `FileIO`s with [[TestCometS3CredentialProvider]] and checks that dropping + * one releases its provider over JNI. Building a `FileIO` makes no storage request, so unlike + * [[CometS3CredentialBridgeSuite]] this needs no S3 server. + */ +class CometS3CredentialReleaseSuite extends CometTestBase { + + private val table = "s3://release-bucket/warehouse/db/t/metadata" + + private def load(metadataFile: String, tag: String, expiresAtMs: Long): Unit = { + val properties = Seq( + "s3.comet.credential.provider.class" -> classOf[TestCometS3CredentialProvider].getName, + "test.instance-tag" -> tag, + "s3.session-token-expires-at-ms" -> expiresAtMs.toString) + NativeBase.loadIcebergFileIOForTesting( + s"$table/$metadataFile", + "release_catalog", + properties.map(_._1).toArray, + properties.map(_._2).toArray) + } + + test("a superseded or cleared Iceberg FileIO releases its provider over JNI") { + assume(NativeBase.isLoaded, "Comet native library not loaded") + NativeBase.clearIcebergFileIOCacheForTesting() + TestCometS3CredentialProvider.reset() + // Close on the releasing thread, so the close count is final when the native call returns. + CometS3CredentialDispatcher.setCloserForTesting((r: Runnable) => r.run()) + try { + val before = CometS3CredentialDispatcher.registeredCount() + + load("00001.metadata.json", "a", expiresAtMs = 1000) + assert(CometS3CredentialDispatcher.registeredCount() == before + 1) + assert(TestCometS3CredentialProvider.initCount.get() == 1) + + // A later commit of the same table, with credentials that expire later, evicts the first + // FileIO, and with it the last reference to the first provider. + load("00002.metadata.json", "b", expiresAtMs = 2000) + assert(TestCometS3CredentialProvider.initCount.get() == 2) + assert(CometS3CredentialDispatcher.registeredCount() == before + 1) + assert(TestCometS3CredentialProvider.closeCount.get() == 1) + + NativeBase.clearIcebergFileIOCacheForTesting() + assert(CometS3CredentialDispatcher.registeredCount() == before) + assert(TestCometS3CredentialProvider.closeCount.get() == 2) + } finally { + CometS3CredentialDispatcher.setCloserForTesting(null) + NativeBase.clearIcebergFileIOCacheForTesting() + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala index 978b1fb89df..ee9f5dd693b 100644 --- a/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/serde/operator/CometIcebergNativeScanSuite.scala @@ -21,7 +21,7 @@ package org.apache.comet.serde.operator import java.io.File import java.lang.reflect.InvocationTargetException -import java.net.URI +import java.net.{URI, URL, URLClassLoader} import java.nio.file.Files import java.util.concurrent.atomic.AtomicInteger @@ -35,11 +35,124 @@ import org.apache.iceberg.expressions.Expressions import org.apache.spark.sql.catalyst.expressions.AttributeReference import org.apache.spark.sql.types.{ArrayType, IntegerType, MapType, StringType, StructType} +import org.apache.comet.CometSparkSessionExtensions.isSpark40Plus import org.apache.comet.iceberg.IcebergReflection /** Unit tests for Iceberg native-scan serde helpers that do not require a Spark session. */ class CometIcebergNativeScanSuite extends AnyFunSuite with Matchers { + private val refreshEndpoint = + Map("client.refresh-credentials-endpoint" -> "/v1/credentials", "uri" -> "https://catalog") + private val providerKey = CometIcebergNativeScan.IcebergCredentialProviderClassKey + + test("vended refresh endpoint selects the vended credential provider") { + CometIcebergNativeScan + .withVendedCredentialProvider(refreshEndpoint, providerAvailable = true) + .get(providerKey) shouldBe Some(CometIcebergNativeScan.VendedCredentialProviderClass) + } + + test("an explicit credential provider, even an empty one, is kept") { + for (explicit <- Seq("com.example.MyProvider", "")) { + CometIcebergNativeScan + .withVendedCredentialProvider( + refreshEndpoint + (providerKey -> explicit), + providerAvailable = true) + .get(providerKey) shouldBe Some(explicit) + } + } + + test("no provider is selected without a usable refresh endpoint") { + for (props <- Seq( + Map.empty[String, String], + Map("client.refresh-credentials-endpoint" -> " "), + refreshEndpoint + ("client.refresh-credentials-enabled" -> "FALSE"))) { + withClue(props) { + CometIcebergNativeScan + .withVendedCredentialProvider(props, providerAvailable = true) shouldBe props + } + } + } + + test("the public overload selects the provider this build ships") { + // Spark 4.x builds ship the provider and the AWS SDK v2 it needs; Spark 3.x builds don't. + val expected = + if (isSpark40Plus) Some(CometIcebergNativeScan.VendedCredentialProviderClass) else None + CometIcebergNativeScan.withVendedCredentialProvider(refreshEndpoint).get(providerKey) shouldBe + expected + } + + // The class loader layouts the driver-side check has to tell apart. Spark 3.x builds don't ship + // the provider, so the check is false there whatever the loaders. + private val sdkPackage = "software.amazon.awssdk." + private def appLoader: ClassLoader = getClass.getClassLoader + + /** Sees everything the test JVM does except the AWS SDK, like Comet beside a user-jar SDK. */ + private def loaderWithoutSdk: ClassLoader = new ClassLoader(appLoader) { + override def loadClass(name: String, resolve: Boolean): Class[_] = + if (name.startsWith(sdkPackage)) throw new ClassNotFoundException(name) + else super.loadClass(name, resolve) + } + + private def jarOf(className: String): URL = + // scalastyle:off classforname + Class.forName(className, false, appLoader).getProtectionDomain.getCodeSource.getLocation + // scalastyle:on classforname + + test("the provider is usable when Comet and Iceberg share one AWS SDK") { + CometIcebergNativeScan.vendedCredentialProviderUsable(appLoader, appLoader) shouldBe + isSpark40Plus + } + + test("the provider is not usable when the only AWS SDK is on the user-jar loader") { + // Iceberg on the context loader resolves the SDK, but Comet's loader can't, so the provider + // would fail to initialize with NoClassDefFoundError. + CometIcebergNativeScan.vendedCredentialProviderUsable(loaderWithoutSdk, appLoader) shouldBe + false + } + + test("the provider is not usable when Iceberg links against another copy of the AWS SDK") { + // A loader with its own iceberg-aws and AWS SDK auth jars, and no parent to share them with, + // as with spark.{driver,executor}.userClassPathFirst and the SDK on both class paths. Spark 3.x + // test class paths need not carry those jars, and the check is false there anyway. + assume(isSpark40Plus) + val isolated = new URLClassLoader( + Array( + jarOf("org.apache.iceberg.aws.AwsClientProperties"), + jarOf("software.amazon.awssdk.auth.credentials.AwsCredentialsProvider"), + jarOf("software.amazon.awssdk.identity.spi.IdentityProvider")), + null) + try { + CometIcebergNativeScan.vendedCredentialProviderUsable(appLoader, isolated) shouldBe false + } finally { + isolated.close() + } + } + + test("a refresh endpoint selects the provider only with a catalog uri") { + // Iceberg's VendedCredentialsProvider rejects a missing uri even for an absolute endpoint. + for (endpoint <- Seq("/v1/credentials", "https://catalog/v1/credentials")) { + withClue(endpoint) { + val props = Map("client.refresh-credentials-endpoint" -> endpoint) + CometIcebergNativeScan + .withVendedCredentialProvider(props, providerAvailable = true) shouldBe props + CometIcebergNativeScan + .withVendedCredentialProvider(props + ("uri" -> " "), providerAvailable = true) + .get(providerKey) shouldBe None + CometIcebergNativeScan + .withVendedCredentialProvider( + props + ("uri" -> "https://catalog"), + providerAvailable = true) + .get(providerKey) shouldBe Some(CometIcebergNativeScan.VendedCredentialProviderClass) + } + } + } + + test("no provider is selected when the build lacks the provider class") { + CometIcebergNativeScan + .withVendedCredentialProvider(refreshEndpoint, providerAvailable = false) shouldBe + refreshEndpoint + } + test("complex type null residuals are not serialized") { // Container predicates stay in the post-scan filter, not the native residual pool. for (dataType <- Seq( diff --git a/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3Provider.java b/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3Provider.java deleted file mode 100644 index 61aa643d9ee..00000000000 --- a/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3Provider.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.comet.cloud.s3; - -import java.time.Instant; -import java.util.Map; - -import org.apache.iceberg.aws.s3.VendedCredentialsProvider; - -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; - -/** - * Example implementation of {@link CometS3CredentialProvider} for Iceberg REST catalogs that vend - * AWS credentials. Wraps Iceberg's {@code VendedCredentialsProvider} so caching and - * refresh-near-expiry come from its {@code CachedSupplier}; Comet adds only the JNI shape and the - * one-shot {@code initialize} call. - * - *

Test scope only, to keep iceberg-aws and AWS SDK v2 off Comet's runtime classpath. Production - * users should copy this into their own jar. - * - *

Spark 4.x build only (Iceberg 1.9.0+ is required for the in-properties caching short-circuit - * in {@code VendedCredentialsProvider}; earlier pins issue an HTTP GET against {@code - * credentials.uri} on every refresh). - * - *

Activation: set {@code spark.sql.catalog..s3.comet.credential.provider.class = - * org.apache.comet.cloud.s3.IcebergRESTVendedS3Provider}. Comet calls {@link #initialize} once per - * catalog with the unfiltered FileIO property bag, which carries {@code credentials.uri} and {@code - * uri} as required by {@code VendedCredentialsProvider.create}. - */ -public final class IcebergRESTVendedS3Provider implements CometS3CredentialProvider { - - private volatile VendedCredentialsProvider provider; - - @Override - public void initialize(Map catalogProperties) { - this.provider = VendedCredentialsProvider.create(catalogProperties); - } - - @Override - public CometS3Credentials getCredentialsForPath(CometS3CredentialContext context) { - VendedCredentialsProvider p = provider; - if (p == null) { - throw new IllegalStateException( - "IcebergRESTVendedS3Provider used before initialize(Map) was called; " - + "Comet should always invoke initialize before getCredentialsForPath"); - } - AwsCredentials c = p.resolveCredentials(); - String sessionToken = null; - long expirationEpochMillis = 0L; - if (c instanceof AwsSessionCredentials) { - AwsSessionCredentials session = (AwsSessionCredentials) c; - sessionToken = session.sessionToken(); - // Publish the vended credential's expiry, so Comet reuses it until shortly before and never - // after. VendedCredentialsProvider's CachedSupplier still decides when to fetch a new one. - expirationEpochMillis = session.expirationTime().map(Instant::toEpochMilli).orElse(0L); - } - return new CometS3Credentials( - c.accessKeyId(), c.secretAccessKey(), sessionToken, expirationEpochMillis); - } -} diff --git a/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3ProviderTest.java b/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3ProviderTest.java deleted file mode 100644 index 5ff5a7517a7..00000000000 --- a/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergRESTVendedS3ProviderTest.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.comet.cloud.s3; - -import java.util.HashMap; -import java.util.Map; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; - -/** - * Unit tests for {@link IcebergRESTVendedS3Provider}. Feeds the wrapped {@code - * VendedCredentialsProvider} a fully-populated {@code s3.*} bag so its {@code CachedSupplier} - * short-circuits without calling the REST {@code credentials.uri} endpoint. Refresh-via-REST - * coverage lives upstream in Iceberg. - */ -public class IcebergRESTVendedS3ProviderTest { - - private static Map staticVendedProps() { - Map props = new HashMap<>(); - // Required by VendedCredentialsProvider.create; not actually contacted in this test. - props.put("uri", "http://localhost:0/catalog"); - props.put("credentials.uri", "http://localhost:0/credentials"); - // Required four keys for the in-properties short-circuit path. - props.put("s3.access-key-id", "AKIA_TEST"); - props.put("s3.secret-access-key", "secret_TEST"); - props.put("s3.session-token", "token_TEST"); - props.put( - "s3.session-token-expires-at-ms", Long.toString(System.currentTimeMillis() + 3_600_000L)); - return props; - } - - @Test - public void initializeThenGetReturnsVendedCredentials() { - IcebergRESTVendedS3Provider p = new IcebergRESTVendedS3Provider(); - Map props = staticVendedProps(); - p.initialize(props); - - CometS3Credentials c = - p.getCredentialsForPath( - new CometS3CredentialContext("bucket", "/k", CometS3AccessMode.READ)); - - assertEquals("AKIA_TEST", c.getAccessKeyId()); - assertEquals("secret_TEST", c.getSecretAccessKey()); - assertEquals("token_TEST", c.getSessionToken()); - // The wrapper publishes the vended credential's expiry, so Comet reuses it until shortly - // before and never after. - assertEquals( - Long.parseLong(props.get("s3.session-token-expires-at-ms")), c.getExpirationEpochMillis()); - } - - @Test - public void getBeforeInitializeThrows() { - IcebergRESTVendedS3Provider p = new IcebergRESTVendedS3Provider(); - assertThrows( - IllegalStateException.class, - () -> - p.getCredentialsForPath( - new CometS3CredentialContext("bucket", "/k", CometS3AccessMode.READ))); - } - - @Test - public void multipleCallsServedByCache() { - IcebergRESTVendedS3Provider p = new IcebergRESTVendedS3Provider(); - p.initialize(staticVendedProps()); - - CometS3Credentials a = - p.getCredentialsForPath(new CometS3CredentialContext("b", "/k", CometS3AccessMode.READ)); - CometS3Credentials b = - p.getCredentialsForPath(new CometS3CredentialContext("b", "/k2", CometS3AccessMode.READ)); - CometS3Credentials c = - p.getCredentialsForPath(new CometS3CredentialContext("b", "/k3", CometS3AccessMode.READ)); - - // CachedSupplier hands out the same identity-equal credential until expiry; we assert by - // value so the test does not depend on AWS SDK internal caching semantics. - assertEquals(a.getAccessKeyId(), b.getAccessKeyId()); - assertEquals(a.getAccessKeyId(), c.getAccessKeyId()); - assertEquals(a.getSecretAccessKey(), b.getSecretAccessKey()); - assertEquals(a.getSessionToken(), b.getSessionToken()); - } -} diff --git a/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProviderTest.java b/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProviderTest.java new file mode 100644 index 00000000000..817d70cabaf --- /dev/null +++ b/spark/src/test/spark-4.x/java/org/apache/comet/cloud/s3/IcebergVendedS3CredentialProviderTest.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.cloud.s3; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import com.sun.net.httpserver.HttpServer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; + +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +/** + * Unit tests for {@link IcebergVendedS3CredentialProvider}. A local HTTP server stands in for the + * REST catalog's credentials endpoint, so the refresh test exercises Iceberg's real {@code + * VendedCredentialsProvider}. + */ +public class IcebergVendedS3CredentialProviderTest { + + private static final CometS3CredentialContext CONTEXT = + new CometS3CredentialContext("bucket", "/warehouse/t/data/f.parquet", CometS3AccessMode.READ); + + private HttpServer server; + private final AtomicInteger refreshCalls = new AtomicInteger(); + private long refreshedExpiryMillis; + + @Before + public void startServer() throws IOException { + refreshedExpiryMillis = System.currentTimeMillis() + 3_600_000L; + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext( + "/v1/credentials", + exchange -> { + refreshCalls.incrementAndGet(); + byte[] body = + ("{\"storage-credentials\":[{\"prefix\":\"s3://bucket\",\"config\":{" + + "\"s3.access-key-id\":\"AKIA_REFRESHED\"," + + "\"s3.secret-access-key\":\"secret_REFRESHED\"," + + "\"s3.session-token\":\"token_REFRESHED\"," + + "\"s3.session-token-expires-at-ms\":\"" + + refreshedExpiryMillis + + "\"}}]}") + .getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, body.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(body); + } + }); + server.start(); + } + + @After + public void stopServer() { + server.stop(0); + } + + private String catalogUri() { + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + private static Map vendedProps(long expiresAtMillis) { + Map props = new HashMap<>(); + props.put("s3.access-key-id", "AKIA_INITIAL"); + props.put("s3.secret-access-key", "secret_INITIAL"); + props.put("s3.session-token", "token_INITIAL"); + props.put("s3.session-token-expires-at-ms", Long.toString(expiresAtMillis)); + return props; + } + + @Test + public void refreshesExpiredVendedCredentialsThroughTheCatalogEndpoint() { + Map props = vendedProps(System.currentTimeMillis() - 60_000L); + props.put("uri", catalogUri()); + props.put("client.refresh-credentials-endpoint", catalogUri() + "/v1/credentials"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_REFRESHED", c.getAccessKeyId()); + assertEquals("secret_REFRESHED", c.getSecretAccessKey()); + assertEquals("token_REFRESHED", c.getSessionToken()); + assertEquals(refreshedExpiryMillis, c.getExpirationEpochMillis()); + assertEquals(1, refreshCalls.get()); + } + + @Test + public void reusesUnexpiredVendedCredentialsWithoutCallingTheEndpoint() { + long expiresAt = System.currentTimeMillis() + 3_600_000L; + Map props = vendedProps(expiresAt); + props.put("uri", catalogUri()); + props.put("client.refresh-credentials-endpoint", catalogUri() + "/v1/credentials"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_INITIAL", c.getAccessKeyId()); + assertEquals("token_INITIAL", c.getSessionToken()); + assertEquals(expiresAt, c.getExpirationEpochMillis()); + assertEquals(0, refreshCalls.get()); + } + + @Test + public void refreshesThroughARelativeEndpointResolvedAgainstTheCatalogUri() { + // REST catalogs usually return the endpoint relative to the catalog uri. + Map props = vendedProps(System.currentTimeMillis() - 60_000L); + props.put("uri", catalogUri()); + props.put("client.refresh-credentials-endpoint", "/v1/credentials"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_REFRESHED", c.getAccessKeyId()); + assertEquals(refreshedExpiryMillis, c.getExpirationEpochMillis()); + assertEquals(1, refreshCalls.get()); + } + + @Test + public void usesStaticVendedCredentialsWhenTheCatalogReturnsNoRefreshEndpoint() { + long expiresAt = System.currentTimeMillis() + 3_600_000L; + Map props = vendedProps(expiresAt); + props.put("uri", catalogUri()); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_INITIAL", c.getAccessKeyId()); + assertEquals("secret_INITIAL", c.getSecretAccessKey()); + assertEquals("token_INITIAL", c.getSessionToken()); + // The vended expiry is reported, so Comet keeps reusing the static credential. + assertEquals(expiresAt, c.getExpirationEpochMillis()); + assertEquals(0, refreshCalls.get()); + } + + @Test + public void honorsRefreshDisabled() { + Map props = vendedProps(System.currentTimeMillis() - 60_000L); + props.put("uri", catalogUri()); + props.put("client.refresh-credentials-endpoint", catalogUri() + "/v1/credentials"); + props.put("client.refresh-credentials-enabled", "false"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_INITIAL", c.getAccessKeyId()); + assertEquals(0, refreshCalls.get()); + } + + @Test + public void staticKeysWithoutSessionTokenReportNoToken() { + Map props = new HashMap<>(); + props.put("s3.access-key-id", "AKIA_STATIC"); + props.put("s3.secret-access-key", "secret_STATIC"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + CometS3Credentials c = p.getCredentialsForPath(CONTEXT); + + assertEquals("AKIA_STATIC", c.getAccessKeyId()); + assertNull(c.getSessionToken()); + assertEquals(0L, c.getExpirationEpochMillis()); + } + + @Test + public void reportsIcebergFailuresWithTheirCause() { + // A refresh endpoint with no catalog uri makes Iceberg throw while building the provider. + Map props = vendedProps(System.currentTimeMillis() + 3_600_000L); + props.put("client.refresh-credentials-endpoint", "/v1/credentials"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + IllegalStateException e = assertThrows(IllegalStateException.class, () -> p.initialize(props)); + assertTrue(e.getMessage(), e.getMessage().contains("failed to build a credentials provider")); + assertTrue(e.getCause() != null && !(e.getCause() instanceof ReflectiveOperationException)); + } + + @Test + public void reportsMissingIcebergAws() { + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + // A loader with no parent cannot see iceberg-aws. + thread.setContextClassLoader(new URLClassLoader(new URL[0], null)); + try { + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> p.initialize(new HashMap<>())); + assertTrue(e.getMessage(), e.getMessage().contains("needs iceberg-aws")); + } finally { + thread.setContextClassLoader(previous); + } + } + + @Test + public void rejectsAnonymousCredentialsWithAHint() { + Map props = new HashMap<>(); + props.put( + "client.credentials-provider", + "software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider"); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> p.getCredentialsForPath(CONTEXT)); + assertTrue(e.getMessage(), e.getMessage().contains("anonymous or empty credentials")); + assertTrue(e.getMessage(), e.getMessage().contains("s3.comet.credential.provider.class")); + } + + /** Stands in for an Iceberg credentials provider that holds a client, as the vended one does. */ + public static final class CloseableCredentialsProvider + implements AwsCredentialsProvider, AutoCloseable { + static final AtomicInteger closeCount = new AtomicInteger(); + + public static CloseableCredentialsProvider create(Map properties) { + return new CloseableCredentialsProvider(); + } + + @Override + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("AKIA_CLOSEABLE", "secret_CLOSEABLE"); + } + + @Override + public void close() { + closeCount.incrementAndGet(); + } + } + + @Test + public void closeClosesTheIcebergProvider() throws Exception { + CloseableCredentialsProvider.closeCount.set(0); + Map props = new HashMap<>(); + props.put("client.credentials-provider", CloseableCredentialsProvider.class.getName()); + + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + p.initialize(props); + assertEquals("AKIA_CLOSEABLE", p.getCredentialsForPath(CONTEXT).getAccessKeyId()); + p.close(); + + assertEquals(1, CloseableCredentialsProvider.closeCount.get()); + } + + @Test + public void getBeforeInitializeThrows() { + IcebergVendedS3CredentialProvider p = new IcebergVendedS3CredentialProvider(); + assertThrows(IllegalStateException.class, () -> p.getCredentialsForPath(CONTEXT)); + } +}