Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions docs/source/user-guide/latest/iceberg.md
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,17 @@ carries the catalog's `hadoop.*` overrides. Also gated explicitly: any `encrypti
`write.object-storage.enabled=true`, and `write.location-provider.impl`. An `io-impl` table
property is ignored, because iceberg-java reads `io-impl` from catalog properties only.

`client.factory` and the `client.assume-role.*` settings are admitted only together, with the
factory set to `org.apache.iceberg.aws.AssumeRoleAwsClientFactory`, both `client.assume-role.arn` and
`client.assume-role.region` set, and the data location written through `S3FileIO` itself, not a
subclass. That region becomes the write region. When `client.assume-role.session-name` is not set,
the native writer names the session `iceberg-aws-<uuid>`, as the factory does. The native writer
calls STS in the assume-role region, while the factory's STS client takes its region from the
default AWS region chain. The write falls back when the factory is combined with a different
`client.region` or `s3.region`, static S3 keys, a credential-provider class, or
`s3.allow-anonymous`, `s3.disable-ec2-metadata` or `s3.disable-config-load` set to a value the
native writer reads as true (`true`, `t`, `1` or `on`, in any case).

Three checks look past properties at the table's instantiated state, because they can be
configured at the catalog level (or by a custom `TableOperations`) without any table or write
property changing: `table.locationProvider()` must be Iceberg's `DefaultLocationProvider`;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ object IcebergReflection extends Logging {
"org.apache.iceberg.LocationProviders$DefaultLocationProvider"
val RESOLVING_FILE_IO = "org.apache.iceberg.io.ResolvingFileIO"
val GCS_FILE_IO = "org.apache.iceberg.gcp.gcs.GCSFileIO"
val S3_FILE_IO = "org.apache.iceberg.aws.s3.S3FileIO"
val PARTITIONING = "org.apache.iceberg.Partitioning"
val SPARK_WRITE = "org.apache.iceberg.spark.source.SparkWrite"
val TABLE_PROPERTIES = "org.apache.iceberg.TableProperties"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -300,9 +300,6 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
"s3.sse.type",
"s3.sse.key",
"s3.sse.md5",
"client.assume-role.arn",
"client.assume-role.external-id",
"client.assume-role.session-name",
"s3.allow-anonymous",
"s3.disable-ec2-metadata",
"s3.disable-config-load",
Expand Down Expand Up @@ -332,6 +329,26 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
// `FileIO.deleteFile`. S3FileIO skips that delete when this is false, so only true is harmless.
private val S3FileIODeleteEnabledProperty = "s3.delete-enabled"

private val S3FileIOClientFactoryProperty = "client.factory"
private val AssumeRoleAwsClientFactoryClass =
"org.apache.iceberg.aws.AssumeRoleAwsClientFactory"
private val AssumeRoleArnProperty = "client.assume-role.arn"
private val AssumeRoleRegionProperty = "client.assume-role.region"
private val AssumeRoleSessionNameProperty = "client.assume-role.session-name"
// iceberg-java ignores these without the factory, so they are admitted only with it.
private val AssumeRoleProperties = Set(
S3FileIOClientFactoryProperty,
AssumeRoleArnProperty,
"client.assume-role.external-id",
AssumeRoleSessionNameProperty,
AssumeRoleRegionProperty)
private val NativeS3RegionProperties = Seq("client.region", "s3.region")
// The factory ignores these, but opendal would use them as AssumeRole's source credentials.
private val AssumeRoleConflictingCredentialProperties =
Seq("s3.access-key-id", "s3.secret-access-key", "s3.session-token")
private val AssumeRoleConflictingCredentialFlags =
Seq("s3.allow-anonymous", "s3.disable-ec2-metadata", "s3.disable-config-load")

// iceberg-java also defines "dsse-kms", but the pinned iceberg-rust S3 backend cannot map
// that mode into an OpenDAL server-side-encryption configuration. Check the value as well as
// the property name so it falls back during planning instead of failing in the native task.
Expand Down Expand Up @@ -1054,21 +1071,35 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
.toMap + (CometS3CredentialProviderClassProperty -> HadoopS3ACredentialAdapterClass)

/** Return unsupported FileIO S3/client property names in deterministic order. */
private[comet] def unsupportedS3FileIOProperties(properties: Map[String, String]): Seq[String] =
unsupportedS3FileIOProperties(properties, IcebergAwsProperties)
private[comet] def unsupportedS3FileIOProperties(
properties: Map[String, String],
s3FileIO: Boolean = false): Seq[String] =
unsupportedS3FileIOProperties(properties, IcebergAwsProperties, s3FileIO)

private[comet] def unsupportedS3FileIOProperties(
properties: Map[String, String],
icebergAwsProperties: IcebergAwsPropertyNames): Seq[String] = {
icebergAwsProperties: IcebergAwsPropertyNames,
s3FileIO: Boolean): Seq[String] = {
val customCredentialProviderConfigured = properties
.get(CometS3CredentialProviderClassProperty)
.exists(_.trim.nonEmpty)

val assumeRole = usesAssumeRoleClientFactory(properties)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Without client.factory, iceberg-java ignores client.assume-role.* (AwsClientFactories.from defaults to DefaultAwsClientFactory, and a HadoopFileIO write goes through S3A). But client.assume-role.arn, external-id and session-name are in SupportedS3FileIOProperties unconditionally, and the test at unsupported(assumeRole - "client.factory") == Seq("client.assume-role.region") shows that an ARN without the factory is still admitted when no region is set. The native writer then assumes the role while the JVM writer would have used the default credentials, so the write happens under a different identity. This predates the PR, but the PR description states the rule and this is the place to enforce it. Consider declining the ARN, external id and session name unless assumeRole holds and the FileIO is S3FileIO.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed here. client.assume-role.arn, external-id and session-name are no longer in SupportedS3FileIOProperties. They are admitted only together with client.factory=AssumeRoleAwsClientFactory, a non-blank arn and region, and S3FileIO, so an ARN without the factory now falls back. I updated the existing supported-keys test and added a case for the assume-role keys without the factory

val assumeRoleConflicts =
if (assumeRole) assumeRoleConflictingProperties(properties) else Seq.empty
val admittedAssumeRoleKeys =
if (s3FileIO && assumeRole && assumeRoleConflicts.isEmpty) {
AssumeRoleProperties
} else {
Set.empty[String]
}

properties.iterator
.filter { case (key, _) => key.startsWith("s3.") || key.startsWith("client.") }
.filter { case (key, value) =>
val unsupportedName = !SupportedS3FileIOProperties.contains(key) &&
!IgnoredS3FileIOProperties.contains(key) &&
!admittedAssumeRoleKeys.contains(key) &&
// PropertyUtil.propertyAsBoolean semantics, as S3FileIOProperties reads it.
!(key == S3FileIODeleteEnabledProperty && java.lang.Boolean.parseBoolean(value))
val unsupportedValue =
Expand All @@ -1079,9 +1110,50 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
}
.map(_._1)
.toSeq
.++(assumeRoleConflicts)
.distinct
.sorted
}

private def assumeRoleConflictingProperties(properties: Map[String, String]): Seq[String] = {
val region = properties(AssumeRoleRegionProperty)
Seq(CometS3CredentialProviderClassProperty).filter(
properties.get(_).exists(_.trim.nonEmpty)) ++
NativeS3RegionProperties.filter(properties.get(_).exists(_ != region)) ++
AssumeRoleConflictingCredentialProperties.filter(properties.contains) ++
AssumeRoleConflictingCredentialFlags.filter(properties.get(_).exists(isTruthy))
}

// iceberg-rust's `is_truthy`, which reads these flags more loosely than iceberg-java.
private def isTruthy(value: String): Boolean =
Set("true", "t", "1", "on").contains(value.toLowerCase(Locale.ROOT))

private def usesAssumeRoleClientFactory(properties: Map[String, String]): Boolean =
properties.get(S3FileIOClientFactoryProperty).contains(AssumeRoleAwsClientFactoryClass) &&
properties.get(AssumeRoleArnProperty).exists(_.trim.nonEmpty) &&
properties.get(AssumeRoleRegionProperty).exists(_.trim.nonEmpty)

// The pinned iceberg-rust has no `client.assume-role.region`, and opendal names the session
// `reqsign` where the factory names it `iceberg-aws-<uuid>`. The gate declines another region.
private[comet] def withAssumeRoleSettings(
fileIOProperties: Map[String, String],
catalogProperties: Map[String, String]): Map[String, String] =
if (usesAssumeRoleClientFactory(fileIOProperties)) {
val region =
if (NativeS3RegionProperties.exists(catalogProperties.contains)) None
else Some("client.region" -> fileIOProperties(AssumeRoleRegionProperty))
val sessionName =
if (catalogProperties.contains(AssumeRoleSessionNameProperty)) None
else Some(AssumeRoleSessionNameProperty -> s"iceberg-aws-${java.util.UUID.randomUUID()}")
catalogProperties ++ region ++ sessionName
} else {
catalogProperties
}

// A subclass can override client creation and ignore the factory.
private[comet] def isS3FileIO(resolvedFileIO: Option[Class[_]]): Boolean =
resolvedFileIO.exists(_.getName == IcebergReflection.ClassNames.S3_FILE_IO)

private val requireSupportedHadoopS3Settings: TriggerRule = ctx =>
s3DataLocation(ctx).flatMap { location =>
val dataBucket = NativeConfig.bucketForUri(new java.net.URI(location), Set.empty)
Expand All @@ -1093,8 +1165,17 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
}

private val requireSupportedS3FileIOProperties: TriggerRule = ctx =>
s3DataLocation(ctx).flatMap { _ =>
unsupportedSettingsReason("S3 FileIO", unsupportedS3FileIOProperties(ctx.fileIOProperties))
s3DataLocation(ctx).flatMap { location =>
// Resolving a ResolvingFileIO's delegate instantiates it.
lazy val s3FileIO = isS3FileIO(
IcebergReflection
.getFileIO(ctx.table)
.flatMap(io => IcebergReflection.resolveFileIOClass(io, location)))
unsupportedSettingsReason(
"S3 FileIO",
unsupportedS3FileIOProperties(
ctx.fileIOProperties,
ctx.fileIOProperties.contains(S3FileIOClientFactoryProperty) && s3FileIO))
}

// HadoopFileIO takes its GCS configuration from `fs.gs.*`, which is not forwarded to the
Expand Down Expand Up @@ -1503,8 +1584,10 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
val storageFileIOProperties =
if (s3aEncryption.isEmpty) fileIOProperties
else fileIOProperties.filter { case (k, _) => !k.startsWith("s3.sse.") }
val catalogProperties = hadoopDerivedProperties ++ storageFileIOProperties ++
s3aEncryption ++ s3aCredentials + CometIcebergNativeScan.ioTimeoutProperty()
val catalogProperties = withAssumeRoleSettings(
fileIOProperties,
hadoopDerivedProperties ++ storageFileIOProperties ++
s3aEncryption ++ s3aCredentials + CometIcebergNativeScan.ioTimeoutProperty())

val common = IcebergWriteProtoTranslation.buildCommon(
catalogProperties = catalogProperties,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -802,9 +802,6 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
"s3.sse.type" -> "kms",
"s3.sse.key" -> "key-id",
"s3.sse.md5" -> "md5",
"client.assume-role.arn" -> "arn",
"client.assume-role.external-id" -> "external",
"client.assume-role.session-name" -> "session",
"s3.allow-anonymous" -> "false",
"s3.disable-ec2-metadata" -> "false",
"s3.disable-config-load" -> "false",
Expand All @@ -822,10 +819,16 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
"s3.access-points.bucket" -> secret,
"s3.remote-signing-enabled" -> secret,
"client.factory" -> secret,
"client.assume-role.arn" -> secret,
"client.assume-role.external-id" -> secret,
"client.assume-role.session-name" -> secret,
"client.credentials-provider" -> secret,
"unrelated.property" -> secret))
assert(
unsupportedFileIO == Seq(
"client.assume-role.arn",
"client.assume-role.external-id",
"client.assume-role.session-name",
"client.credentials-provider",
"client.factory",
"s3.access-points.bucket",
Expand Down Expand Up @@ -863,6 +866,112 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
unsupportedWithCustomProvider)
}

test("S3 FileIO admits AssumeRoleAwsClientFactory with its role and region") {
val factory = "org.apache.iceberg.aws.AssumeRoleAwsClientFactory"
val assumeRole = Map(
"client.factory" -> factory,
"client.assume-role.arn" -> "arn:aws:iam::123456789012:role/data",
"client.assume-role.region" -> "eu-west-1")
val declined =
Seq("client.assume-role.arn", "client.assume-role.region", "client.factory")
def unsupported(properties: Map[String, String]): Seq[String] =
CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, s3FileIO = true)

assert(unsupported(assumeRole).isEmpty)
assert(unsupported(assumeRole + ("client.region" -> "eu-west-1")).isEmpty)
assert(unsupported(assumeRole + ("s3.region" -> "eu-west-1")).isEmpty)
assert(unsupported(assumeRole + ("s3.allow-anonymous" -> "false")).isEmpty)
assert(
unsupported(
assumeRole ++ Map(
"client.assume-role.external-id" -> "external",
"client.assume-role.session-name" -> "session")).isEmpty)

assert(
CometIcebergNativeWrite.unsupportedS3FileIOProperties(assumeRole, s3FileIO = false) ==
declined)

assert(
unsupported(assumeRole + ("client.region" -> "us-east-1")) == declined :+ "client.region")
assert(unsupported(assumeRole + ("s3.region" -> "us-east-1")) == declined :+ "s3.region")

Seq(
"s3.access-key-id" -> "access",
"s3.secret-access-key" -> "secret",
"s3.session-token" -> "token",
"s3.allow-anonymous" -> "true",
"s3.disable-ec2-metadata" -> "true",
"s3.disable-config-load" -> "true",
"s3.allow-anonymous" -> "1",
"s3.disable-ec2-metadata" -> "T",
"s3.disable-config-load" -> "on").foreach { case (key, value) =>
assert(unsupported(assumeRole + (key -> value)) == (declined :+ key).sorted, key)
}

assert(
unsupported(assumeRole + ("client.factory" -> "com.example.CustomAwsClientFactory")) ==
declined)
assert(unsupported(assumeRole + ("client.factory" -> s" $factory")) == declined)
assert(
unsupported(assumeRole - "client.assume-role.arn") ==
Seq("client.assume-role.region", "client.factory"))
assert(unsupported(assumeRole + ("client.assume-role.arn" -> " ")) == declined)
assert(unsupported(assumeRole + ("client.assume-role.region" -> " ")) == declined)
assert(
unsupported(
assumeRole - "client.factory" ++ Map(
"client.assume-role.external-id" -> "external",
"client.assume-role.session-name" -> "session")) ==
Seq(
"client.assume-role.arn",
"client.assume-role.external-id",
"client.assume-role.region",
"client.assume-role.session-name"))

assert(
unsupported(
assumeRole ++ Map(
"client.assume-role.tags.team" -> "data",
"client.assume-role.timeout-sec" -> "900")) ==
Seq("client.assume-role.tags.team", "client.assume-role.timeout-sec"))

assert(
unsupported(assumeRole + ("s3.comet.credential.provider.class" -> "provider")) ==
declined :+ "s3.comet.credential.provider.class")
}

test("the native writer takes AssumeRoleAwsClientFactory's region and session name") {
val assumeRole = Map(
"client.factory" -> "org.apache.iceberg.aws.AssumeRoleAwsClientFactory",
"client.assume-role.arn" -> "arn:aws:iam::123456789012:role/data",
"client.assume-role.region" -> "eu-west-1")
def settings(properties: Map[String, String]): Map[String, String] =
CometIcebergNativeWrite.withAssumeRoleSettings(properties, properties)

val defaulted = settings(assumeRole)
assert(
defaulted - "client.assume-role.session-name" == assumeRole + ("client.region" -> "eu-west-1"))
assert(
defaulted("client.assume-role.session-name")
.matches("iceberg-aws-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
defaulted)

val configured = assumeRole ++ Map(
"s3.region" -> "eu-west-1",
"client.assume-role.session-name" -> "session")
assert(settings(configured) == configured)

val noFactory = assumeRole - "client.factory"
assert(settings(noFactory) == noFactory)
}

test("only S3FileIO itself takes AssumeRoleAwsClientFactory") {
assert(CometIcebergNativeWrite.isS3FileIO(Some(classOf[S3FileIO])))
assert(!CometIcebergNativeWrite.isS3FileIO(Some(classOf[DetectionS3FileIOSubclass])))
assert(!CometIcebergNativeWrite.isS3FileIO(Some(classOf[HadoopFileIO])))
assert(!CometIcebergNativeWrite.isS3FileIO(None))
}

test("a longer dotted bucket is not the target bucket") {
val hadoopConf = new Configuration(false)
hadoopConf.set("fs.s3a.bucket.target.access.key", "access")
Expand Down Expand Up @@ -903,7 +1012,7 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
loadFailures.foreach { failure =>
val names = CometIcebergNativeWrite.icebergAwsPropertyNames(_ => throw failure)
val unsupported =
CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, names)
CometIcebergNativeWrite.unsupportedS3FileIOProperties(properties, names, false)
assert(unsupported == expected, unsupported)
assert(!unsupported.exists(_.contains(secret)), unsupported)
}
Expand Down Expand Up @@ -2406,6 +2515,8 @@ class DetectionDelegatingLocationProvider(delegate: LocationProvider) extends Lo
}

/** A real S3FileIO for data, with local metadata managed by HadoopCatalog. Planning only. */
class DetectionS3FileIOSubclass extends S3FileIO

class DetectionS3FileIOHadoopCatalog extends HadoopCatalog {
override protected def newTableOps(identifier: TableIdentifier): TableOperations =
new DetectionS3FileIOTableOperations(super.newTableOps(identifier))
Expand Down