diff --git a/docs/source/user-guide/latest/iceberg.md b/docs/source/user-guide/latest/iceberg.md index b856772cd96..402ef23211a 100644 --- a/docs/source/user-guide/latest/iceberg.md +++ b/docs/source/user-guide/latest/iceberg.md @@ -571,6 +571,17 @@ carries the catalog's `hadoop.*` overrides. Also gated explicitly: any `encrypti `write.object-storage.enabled=true`, and `write.location-provider.impl`. An `io-impl` table property is ignored, because iceberg-java reads `io-impl` from catalog properties only. +`client.factory` and the `client.assume-role.*` settings are admitted only together, with the +factory set to `org.apache.iceberg.aws.AssumeRoleAwsClientFactory`, both `client.assume-role.arn` and +`client.assume-role.region` set, and the data location written through `S3FileIO` itself, not a +subclass. That region becomes the write region. When `client.assume-role.session-name` is not set, +the native writer names the session `iceberg-aws-`, as the factory does. The native writer +calls STS in the assume-role region, while the factory's STS client takes its region from the +default AWS region chain. The write falls back when the factory is combined with a different +`client.region` or `s3.region`, static S3 keys, a credential-provider class, or +`s3.allow-anonymous`, `s3.disable-ec2-metadata` or `s3.disable-config-load` set to a value the +native writer reads as true (`true`, `t`, `1` or `on`, in any case). + Three checks look past properties at the table's instantiated state, because they can be configured at the catalog level (or by a custom `TableOperations`) without any table or write property changing: `table.locationProvider()` must be Iceberg's `DefaultLocationProvider`; diff --git a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala index 79ca850e63c..b9f28067edc 100644 --- a/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala +++ b/spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala @@ -63,6 +63,7 @@ object IcebergReflection extends Logging { "org.apache.iceberg.LocationProviders$DefaultLocationProvider" val RESOLVING_FILE_IO = "org.apache.iceberg.io.ResolvingFileIO" val GCS_FILE_IO = "org.apache.iceberg.gcp.gcs.GCSFileIO" + val S3_FILE_IO = "org.apache.iceberg.aws.s3.S3FileIO" val PARTITIONING = "org.apache.iceberg.Partitioning" val SPARK_WRITE = "org.apache.iceberg.spark.source.SparkWrite" val TABLE_PROPERTIES = "org.apache.iceberg.TableProperties" diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala index 773ab11d800..f8a0636279e 100644 --- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala +++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala @@ -300,9 +300,6 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { "s3.sse.type", "s3.sse.key", "s3.sse.md5", - "client.assume-role.arn", - "client.assume-role.external-id", - "client.assume-role.session-name", "s3.allow-anonymous", "s3.disable-ec2-metadata", "s3.disable-config-load", @@ -332,6 +329,26 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { // `FileIO.deleteFile`. S3FileIO skips that delete when this is false, so only true is harmless. private val S3FileIODeleteEnabledProperty = "s3.delete-enabled" + private val S3FileIOClientFactoryProperty = "client.factory" + private val AssumeRoleAwsClientFactoryClass = + "org.apache.iceberg.aws.AssumeRoleAwsClientFactory" + private val AssumeRoleArnProperty = "client.assume-role.arn" + private val AssumeRoleRegionProperty = "client.assume-role.region" + private val AssumeRoleSessionNameProperty = "client.assume-role.session-name" + // iceberg-java ignores these without the factory, so they are admitted only with it. + private val AssumeRoleProperties = Set( + S3FileIOClientFactoryProperty, + AssumeRoleArnProperty, + "client.assume-role.external-id", + AssumeRoleSessionNameProperty, + AssumeRoleRegionProperty) + private val NativeS3RegionProperties = Seq("client.region", "s3.region") + // The factory ignores these, but opendal would use them as AssumeRole's source credentials. + private val AssumeRoleConflictingCredentialProperties = + Seq("s3.access-key-id", "s3.secret-access-key", "s3.session-token") + private val AssumeRoleConflictingCredentialFlags = + Seq("s3.allow-anonymous", "s3.disable-ec2-metadata", "s3.disable-config-load") + // iceberg-java also defines "dsse-kms", but the pinned iceberg-rust S3 backend cannot map // that mode into an OpenDAL server-side-encryption configuration. Check the value as well as // the property name so it falls back during planning instead of failing in the native task. @@ -1054,21 +1071,35 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { .toMap + (CometS3CredentialProviderClassProperty -> HadoopS3ACredentialAdapterClass) /** Return unsupported FileIO S3/client property names in deterministic order. */ - private[comet] def unsupportedS3FileIOProperties(properties: Map[String, String]): Seq[String] = - unsupportedS3FileIOProperties(properties, IcebergAwsProperties) + private[comet] def unsupportedS3FileIOProperties( + properties: Map[String, String], + s3FileIO: Boolean = false): Seq[String] = + unsupportedS3FileIOProperties(properties, IcebergAwsProperties, s3FileIO) private[comet] def unsupportedS3FileIOProperties( properties: Map[String, String], - icebergAwsProperties: IcebergAwsPropertyNames): Seq[String] = { + icebergAwsProperties: IcebergAwsPropertyNames, + s3FileIO: Boolean): Seq[String] = { val customCredentialProviderConfigured = properties .get(CometS3CredentialProviderClassProperty) .exists(_.trim.nonEmpty) + val assumeRole = usesAssumeRoleClientFactory(properties) + val assumeRoleConflicts = + if (assumeRole) assumeRoleConflictingProperties(properties) else Seq.empty + val admittedAssumeRoleKeys = + if (s3FileIO && assumeRole && assumeRoleConflicts.isEmpty) { + AssumeRoleProperties + } else { + Set.empty[String] + } + properties.iterator .filter { case (key, _) => key.startsWith("s3.") || key.startsWith("client.") } .filter { case (key, value) => val unsupportedName = !SupportedS3FileIOProperties.contains(key) && !IgnoredS3FileIOProperties.contains(key) && + !admittedAssumeRoleKeys.contains(key) && // PropertyUtil.propertyAsBoolean semantics, as S3FileIOProperties reads it. !(key == S3FileIODeleteEnabledProperty && java.lang.Boolean.parseBoolean(value)) val unsupportedValue = @@ -1079,9 +1110,50 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } .map(_._1) .toSeq + .++(assumeRoleConflicts) + .distinct .sorted } + private def assumeRoleConflictingProperties(properties: Map[String, String]): Seq[String] = { + val region = properties(AssumeRoleRegionProperty) + Seq(CometS3CredentialProviderClassProperty).filter( + properties.get(_).exists(_.trim.nonEmpty)) ++ + NativeS3RegionProperties.filter(properties.get(_).exists(_ != region)) ++ + AssumeRoleConflictingCredentialProperties.filter(properties.contains) ++ + AssumeRoleConflictingCredentialFlags.filter(properties.get(_).exists(isTruthy)) + } + + // iceberg-rust's `is_truthy`, which reads these flags more loosely than iceberg-java. + private def isTruthy(value: String): Boolean = + Set("true", "t", "1", "on").contains(value.toLowerCase(Locale.ROOT)) + + private def usesAssumeRoleClientFactory(properties: Map[String, String]): Boolean = + properties.get(S3FileIOClientFactoryProperty).contains(AssumeRoleAwsClientFactoryClass) && + properties.get(AssumeRoleArnProperty).exists(_.trim.nonEmpty) && + properties.get(AssumeRoleRegionProperty).exists(_.trim.nonEmpty) + + // The pinned iceberg-rust has no `client.assume-role.region`, and opendal names the session + // `reqsign` where the factory names it `iceberg-aws-`. The gate declines another region. + private[comet] def withAssumeRoleSettings( + fileIOProperties: Map[String, String], + catalogProperties: Map[String, String]): Map[String, String] = + if (usesAssumeRoleClientFactory(fileIOProperties)) { + val region = + if (NativeS3RegionProperties.exists(catalogProperties.contains)) None + else Some("client.region" -> fileIOProperties(AssumeRoleRegionProperty)) + val sessionName = + if (catalogProperties.contains(AssumeRoleSessionNameProperty)) None + else Some(AssumeRoleSessionNameProperty -> s"iceberg-aws-${java.util.UUID.randomUUID()}") + catalogProperties ++ region ++ sessionName + } else { + catalogProperties + } + + // A subclass can override client creation and ignore the factory. + private[comet] def isS3FileIO(resolvedFileIO: Option[Class[_]]): Boolean = + resolvedFileIO.exists(_.getName == IcebergReflection.ClassNames.S3_FILE_IO) + private val requireSupportedHadoopS3Settings: TriggerRule = ctx => s3DataLocation(ctx).flatMap { location => val dataBucket = NativeConfig.bucketForUri(new java.net.URI(location), Set.empty) @@ -1093,8 +1165,17 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { } private val requireSupportedS3FileIOProperties: TriggerRule = ctx => - s3DataLocation(ctx).flatMap { _ => - unsupportedSettingsReason("S3 FileIO", unsupportedS3FileIOProperties(ctx.fileIOProperties)) + s3DataLocation(ctx).flatMap { location => + // Resolving a ResolvingFileIO's delegate instantiates it. + lazy val s3FileIO = isS3FileIO( + IcebergReflection + .getFileIO(ctx.table) + .flatMap(io => IcebergReflection.resolveFileIOClass(io, location))) + unsupportedSettingsReason( + "S3 FileIO", + unsupportedS3FileIOProperties( + ctx.fileIOProperties, + ctx.fileIOProperties.contains(S3FileIOClientFactoryProperty) && s3FileIO)) } // HadoopFileIO takes its GCS configuration from `fs.gs.*`, which is not forwarded to the @@ -1503,8 +1584,10 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] { val storageFileIOProperties = if (s3aEncryption.isEmpty) fileIOProperties else fileIOProperties.filter { case (k, _) => !k.startsWith("s3.sse.") } - val catalogProperties = hadoopDerivedProperties ++ storageFileIOProperties ++ - s3aEncryption ++ s3aCredentials + CometIcebergNativeScan.ioTimeoutProperty() + val catalogProperties = withAssumeRoleSettings( + fileIOProperties, + hadoopDerivedProperties ++ storageFileIOProperties ++ + s3aEncryption ++ s3aCredentials + CometIcebergNativeScan.ioTimeoutProperty()) val common = IcebergWriteProtoTranslation.buildCommon( catalogProperties = catalogProperties, diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala index 9bbacfb0df0..ca746199241 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala @@ -802,9 +802,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes "s3.sse.type" -> "kms", "s3.sse.key" -> "key-id", "s3.sse.md5" -> "md5", - "client.assume-role.arn" -> "arn", - "client.assume-role.external-id" -> "external", - "client.assume-role.session-name" -> "session", "s3.allow-anonymous" -> "false", "s3.disable-ec2-metadata" -> "false", "s3.disable-config-load" -> "false", @@ -822,10 +819,16 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes "s3.access-points.bucket" -> secret, "s3.remote-signing-enabled" -> secret, "client.factory" -> secret, + "client.assume-role.arn" -> secret, + "client.assume-role.external-id" -> secret, + "client.assume-role.session-name" -> secret, "client.credentials-provider" -> secret, "unrelated.property" -> secret)) assert( unsupportedFileIO == Seq( + "client.assume-role.arn", + "client.assume-role.external-id", + "client.assume-role.session-name", "client.credentials-provider", "client.factory", "s3.access-points.bucket", @@ -863,6 +866,112 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes unsupportedWithCustomProvider) } + test("S3 FileIO admits AssumeRoleAwsClientFactory with its role and region") { + val factory = "org.apache.iceberg.aws.AssumeRoleAwsClientFactory" + val assumeRole = Map( + "client.factory" -> factory, + "client.assume-role.arn" -> "arn:aws:iam::123456789012:role/data", + "client.assume-role.region" -> "eu-west-1") + val declined = + Seq("client.assume-role.arn", "client.assume-role.region", "client.factory") + def unsupported(properties: Map[String, String]): Seq[String] = + CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, s3FileIO = true) + + assert(unsupported(assumeRole).isEmpty) + assert(unsupported(assumeRole + ("client.region" -> "eu-west-1")).isEmpty) + assert(unsupported(assumeRole + ("s3.region" -> "eu-west-1")).isEmpty) + assert(unsupported(assumeRole + ("s3.allow-anonymous" -> "false")).isEmpty) + assert( + unsupported( + assumeRole ++ Map( + "client.assume-role.external-id" -> "external", + "client.assume-role.session-name" -> "session")).isEmpty) + + assert( + CometIcebergNativeWrite.unsupportedS3FileIOProperties(assumeRole, s3FileIO = false) == + declined) + + assert( + unsupported(assumeRole + ("client.region" -> "us-east-1")) == declined :+ "client.region") + assert(unsupported(assumeRole + ("s3.region" -> "us-east-1")) == declined :+ "s3.region") + + Seq( + "s3.access-key-id" -> "access", + "s3.secret-access-key" -> "secret", + "s3.session-token" -> "token", + "s3.allow-anonymous" -> "true", + "s3.disable-ec2-metadata" -> "true", + "s3.disable-config-load" -> "true", + "s3.allow-anonymous" -> "1", + "s3.disable-ec2-metadata" -> "T", + "s3.disable-config-load" -> "on").foreach { case (key, value) => + assert(unsupported(assumeRole + (key -> value)) == (declined :+ key).sorted, key) + } + + assert( + unsupported(assumeRole + ("client.factory" -> "com.example.CustomAwsClientFactory")) == + declined) + assert(unsupported(assumeRole + ("client.factory" -> s" $factory")) == declined) + assert( + unsupported(assumeRole - "client.assume-role.arn") == + Seq("client.assume-role.region", "client.factory")) + assert(unsupported(assumeRole + ("client.assume-role.arn" -> " ")) == declined) + assert(unsupported(assumeRole + ("client.assume-role.region" -> " ")) == declined) + assert( + unsupported( + assumeRole - "client.factory" ++ Map( + "client.assume-role.external-id" -> "external", + "client.assume-role.session-name" -> "session")) == + Seq( + "client.assume-role.arn", + "client.assume-role.external-id", + "client.assume-role.region", + "client.assume-role.session-name")) + + assert( + unsupported( + assumeRole ++ Map( + "client.assume-role.tags.team" -> "data", + "client.assume-role.timeout-sec" -> "900")) == + Seq("client.assume-role.tags.team", "client.assume-role.timeout-sec")) + + assert( + unsupported(assumeRole + ("s3.comet.credential.provider.class" -> "provider")) == + declined :+ "s3.comet.credential.provider.class") + } + + test("the native writer takes AssumeRoleAwsClientFactory's region and session name") { + val assumeRole = Map( + "client.factory" -> "org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + "client.assume-role.arn" -> "arn:aws:iam::123456789012:role/data", + "client.assume-role.region" -> "eu-west-1") + def settings(properties: Map[String, String]): Map[String, String] = + CometIcebergNativeWrite.withAssumeRoleSettings(properties, properties) + + val defaulted = settings(assumeRole) + assert( + defaulted - "client.assume-role.session-name" == assumeRole + ("client.region" -> "eu-west-1")) + assert( + defaulted("client.assume-role.session-name") + .matches("iceberg-aws-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"), + defaulted) + + val configured = assumeRole ++ Map( + "s3.region" -> "eu-west-1", + "client.assume-role.session-name" -> "session") + assert(settings(configured) == configured) + + val noFactory = assumeRole - "client.factory" + assert(settings(noFactory) == noFactory) + } + + test("only S3FileIO itself takes AssumeRoleAwsClientFactory") { + assert(CometIcebergNativeWrite.isS3FileIO(Some(classOf[S3FileIO]))) + assert(!CometIcebergNativeWrite.isS3FileIO(Some(classOf[DetectionS3FileIOSubclass]))) + assert(!CometIcebergNativeWrite.isS3FileIO(Some(classOf[HadoopFileIO]))) + assert(!CometIcebergNativeWrite.isS3FileIO(None)) + } + test("a longer dotted bucket is not the target bucket") { val hadoopConf = new Configuration(false) hadoopConf.set("fs.s3a.bucket.target.access.key", "access") @@ -903,7 +1012,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes loadFailures.foreach { failure => val names = CometIcebergNativeWrite.icebergAwsPropertyNames(_ => throw failure) val unsupported = - CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, names) + CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, names, false) assert(unsupported == expected, unsupported) assert(!unsupported.exists(_.contains(secret)), unsupported) } @@ -2406,6 +2515,8 @@ class DetectionDelegatingLocationProvider(delegate: LocationProvider) extends Lo } /** A real S3FileIO for data, with local metadata managed by HadoopCatalog. Planning only. */ +class DetectionS3FileIOSubclass extends S3FileIO + class DetectionS3FileIOHadoopCatalog extends HadoopCatalog { override protected def newTableOps(identifier: TableIdentifier): TableOperations = new DetectionS3FileIOTableOperations(super.newTableOps(identifier))