diff --git a/changelog.d/implicit-only-federation-authentication.md b/changelog.d/implicit-only-federation-authentication.md new file mode 100644 index 00000000000..36052525e3a --- /dev/null +++ b/changelog.d/implicit-only-federation-authentication.md @@ -0,0 +1,8 @@ +### Fixes + +- Creating an external catalog with a `HADOOP`, `HIVE` or `BIGQUERY` connection and an + authentication type other than `IMPLICIT` now returns `400 Bad Request`. These connection types + only support `IMPLICIT` authentication; previously such a catalog was created, its secret was + stored, and every later request against it failed with `500 Internal Server Error`. A catalog + that was created with such a combination before this change cannot serve any request and is now + also rejected when updated; drop and recreate it with `IMPLICIT` authentication. diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidator.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidator.java new file mode 100644 index 00000000000..5fde572b8c9 --- /dev/null +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidator.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import jakarta.enterprise.context.ApplicationScoped; +import java.util.Set; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.ExternalCatalog; + +/** + * Rejects federated catalogs whose connection type only works with {@code IMPLICIT} authentication + * but that request another authentication type. The Hadoop, Hive and BigQuery Metastore federation + * factories refuse any other type when the catalog is first used, so such a catalog could be + * created but never used. + */ +@ApplicationScoped +public class ImplicitOnlyConnectionValidator implements CatalogConfigValidator { + + private static final Set IMPLICIT_ONLY = + Set.of( + ConnectionConfigInfo.ConnectionTypeEnum.HADOOP, + ConnectionConfigInfo.ConnectionTypeEnum.HIVE, + ConnectionConfigInfo.ConnectionTypeEnum.BIGQUERY); + + @Override + public void validate(Catalog catalog) { + if (!(catalog instanceof ExternalCatalog externalCatalog)) { + return; + } + ConnectionConfigInfo connection = externalCatalog.getConnectionConfigInfo(); + if (connection == null || !IMPLICIT_ONLY.contains(connection.getConnectionType())) { + return; + } + // A missing value is reported by the request validation that requires it. + AuthenticationParameters authentication = connection.getAuthenticationParameters(); + if (authentication == null + || authentication.getAuthenticationType() + == AuthenticationParameters.AuthenticationTypeEnum.IMPLICIT) { + return; + } + throw new IllegalArgumentException( + connection.getConnectionType().name() + + " connections only support IMPLICIT authentication, but " + + authentication.getAuthenticationType().name() + + " was requested"); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidatorTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidatorTest.java new file mode 100644 index 00000000000..4cc4f12f272 --- /dev/null +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ImplicitOnlyConnectionValidatorTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.admin; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.List; +import org.apache.polaris.core.admin.model.AuthenticationParameters; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.Catalog; +import org.apache.polaris.core.admin.model.CatalogProperties; +import org.apache.polaris.core.admin.model.ConnectionConfigInfo; +import org.apache.polaris.core.admin.model.ExternalCatalog; +import org.apache.polaris.core.admin.model.FileStorageConfigInfo; +import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo; +import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; +import org.apache.polaris.core.admin.model.ImplicitAuthenticationParameters; +import org.apache.polaris.core.admin.model.PolarisCatalog; +import org.apache.polaris.core.admin.model.StorageConfigInfo; +import org.junit.jupiter.api.Test; + +class ImplicitOnlyConnectionValidatorTest { + private final ImplicitOnlyConnectionValidator validator = new ImplicitOnlyConnectionValidator(); + + @Test + void implicitAuthenticationIsAccepted() { + assertThatCode(() -> validator.validate(hadoopCatalog(implicit()))).doesNotThrowAnyException(); + } + + @Test + void otherAuthenticationIsRejected() { + assertThatThrownBy(() -> validator.validate(hadoopCatalog(bearer()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + "HADOOP connections only support IMPLICIT authentication, but BEARER was requested"); + } + + @Test + void missingAuthenticationIsLeftToRequestValidation() { + assertThatCode(() -> validator.validate(hadoopCatalog(null))).doesNotThrowAnyException(); + } + + @Test + void otherConnectionTypesAreNotRestricted() { + Catalog catalog = + externalCatalog( + IcebergRestConnectionConfigInfo.builder( + ConnectionConfigInfo.ConnectionTypeEnum.ICEBERG_REST) + .setUri("https://remote.example.com/api/catalog") + .setRemoteCatalogName("remote") + .setAuthenticationParameters(bearer()) + .build()); + + assertThatCode(() -> validator.validate(catalog)).doesNotThrowAnyException(); + } + + @Test + void catalogsWithoutConnectionAreIgnored() { + Catalog internal = + PolarisCatalog.builder() + .setType(Catalog.TypeEnum.INTERNAL) + .setName("internal") + .setProperties(new CatalogProperties("file:///tmp/internal")) + .setStorageConfigInfo(fileStorage()) + .build(); + + assertThatCode(() -> validator.validate(internal)).doesNotThrowAnyException(); + assertThatCode(() -> validator.validate(externalCatalog(null))).doesNotThrowAnyException(); + } + + private static Catalog hadoopCatalog(AuthenticationParameters authenticationParameters) { + return externalCatalog( + HadoopConnectionConfigInfo.builder(ConnectionConfigInfo.ConnectionTypeEnum.HADOOP) + .setWarehouse("file:///tmp/warehouse") + .setAuthenticationParameters(authenticationParameters) + .build()); + } + + private static Catalog externalCatalog(ConnectionConfigInfo connection) { + return ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName("external") + .setProperties(new CatalogProperties("file:///tmp/external")) + .setStorageConfigInfo(fileStorage()) + .setConnectionConfigInfo(connection) + .build(); + } + + private static StorageConfigInfo fileStorage() { + return FileStorageConfigInfo.builder() + .setStorageType(StorageConfigInfo.StorageTypeEnum.FILE) + .setAllowedLocations(List.of("file:///tmp/")) + .build(); + } + + private static AuthenticationParameters implicit() { + return ImplicitAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.IMPLICIT) + .build(); + } + + private static AuthenticationParameters bearer() { + return BearerAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) + .setBearerToken("some-token") + .build(); + } +} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index fd9971d150f..17920c6c06c 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -26,6 +26,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.function.Supplier; @@ -35,6 +36,8 @@ import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.admin.model.AuthenticationParameters; import org.apache.polaris.core.admin.model.AwsStorageConfigInfo; +import org.apache.polaris.core.admin.model.BearerAuthenticationParameters; +import org.apache.polaris.core.admin.model.BigQueryMetastoreConnectionConfigInfo; import org.apache.polaris.core.admin.model.Catalog; import org.apache.polaris.core.admin.model.CatalogProperties; import org.apache.polaris.core.admin.model.ConnectionConfigInfo; @@ -43,6 +46,8 @@ import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpAuthenticationParameters; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; +import org.apache.polaris.core.admin.model.HadoopConnectionConfigInfo; +import org.apache.polaris.core.admin.model.HiveConnectionConfigInfo; import org.apache.polaris.core.admin.model.IcebergRestConnectionConfigInfo; import org.apache.polaris.core.admin.model.OAuthClientCredentialsParameters; import org.apache.polaris.core.admin.model.PolarisCatalog; @@ -76,6 +81,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; @@ -577,7 +583,9 @@ private PolarisAdminService setupPolarisAdminService( ReservedProperties.NONE, services.vendingMechanisms(), new CatalogConfigValidators( - List.of(new BigLakeCatalogValidator(callContext.getRealmConfig())))); + List.of( + new BigLakeCatalogValidator(callContext.getRealmConfig()), + new ImplicitOnlyConnectionValidator()))); } private Catalog createBigLakeCatalog( @@ -618,6 +626,73 @@ private StorageConfigInfo createBigLakeStorageConfig( .build(); } + /** + * The Hadoop, Hive and BigQuery Metastore federation factories only support IMPLICIT + * authentication. A catalog with any other authentication type could never be used, so it is + * rejected at creation and never persisted. + */ + @ParameterizedTest + @EnumSource( + value = ConnectionConfigInfo.ConnectionTypeEnum.class, + names = {"HADOOP", "HIVE", "BIGQUERY"}) + public void testCreateImplicitOnlyFederatedCatalogWithOtherAuthenticationIsRejected( + ConnectionConfigInfo.ConnectionTypeEnum connectionType) { + PolarisMetaStoreManager metaStoreManager = Mockito.spy(services.metaStoreManager()); + PolarisCallContext callContext = services.newCallContext(); + PolarisAdminService polarisAdminService = + setupPolarisAdminService(metaStoreManager, callContext); + AuthenticationParameters bearer = + BearerAuthenticationParameters.builder() + .setAuthenticationType(AuthenticationParameters.AuthenticationTypeEnum.BEARER) + .setBearerToken("some-token") + .build(); + Catalog catalog = + ExternalCatalog.builder() + .setType(Catalog.TypeEnum.EXTERNAL) + .setName("federated_" + connectionType.name().toLowerCase(Locale.ROOT)) + .setProperties(new CatalogProperties("s3://bucket/path/to/data")) + .setStorageConfigInfo( + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::123456789012:role/my-role") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of("s3://bucket/path/to/data")) + .build()) + .setConnectionConfigInfo(implicitOnlyConnection(connectionType, bearer)) + .build(); + + assertThatThrownBy(() -> polarisAdminService.createCatalog(new CreateCatalogRequest(catalog))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage( + connectionType.name() + + " connections only support IMPLICIT authentication, but BEARER was requested"); + Mockito.verify(metaStoreManager, Mockito.never()) + .createCatalog(Mockito.any(), Mockito.any(), Mockito.any()); + } + + private static ConnectionConfigInfo implicitOnlyConnection( + ConnectionConfigInfo.ConnectionTypeEnum connectionType, + AuthenticationParameters authenticationParameters) { + return switch (connectionType) { + case HADOOP -> + HadoopConnectionConfigInfo.builder(connectionType) + .setWarehouse("s3://bucket/path/to/data") + .setAuthenticationParameters(authenticationParameters) + .build(); + case HIVE -> + HiveConnectionConfigInfo.builder(connectionType) + .setUri("thrift://hms:9083") + .setWarehouse("s3://bucket/path/to/data") + .setAuthenticationParameters(authenticationParameters) + .build(); + case BIGQUERY -> + BigQueryMetastoreConnectionConfigInfo.builder(connectionType) + .setProperties(Map.of("gcp.bigquery.project-id", "p", "warehouse", "w")) + .setAuthenticationParameters(authenticationParameters) + .build(); + default -> throw new IllegalArgumentException(connectionType.name()); + }; + } + private PrincipalEntity createPrincipal( PolarisMetaStoreManager metaStoreManager, PolarisCallContext callContext, String name) { return new PrincipalEntity.Builder() diff --git a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java index f4b779a8a1b..cd33210ab0c 100644 --- a/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java +++ b/runtime/service/src/testFixtures/java/org/apache/polaris/service/TestServices.java @@ -81,6 +81,7 @@ import org.apache.polaris.core.storage.cache.StorageCredentialCacheConfig; import org.apache.polaris.service.admin.BigLakeCatalogValidator; import org.apache.polaris.service.admin.CatalogConfigValidators; +import org.apache.polaris.service.admin.ImplicitOnlyConnectionValidator; import org.apache.polaris.service.admin.PolarisAdminService; import org.apache.polaris.service.admin.PolarisServiceImpl; import org.apache.polaris.service.admin.api.PolarisCatalogsApi; @@ -613,7 +614,9 @@ public GenericTableCatalogHandler createHandler( }; CatalogConfigValidators catalogConfigValidators = - new CatalogConfigValidators(List.of(new BigLakeCatalogValidator(realmConfig))); + new CatalogConfigValidators( + List.of( + new BigLakeCatalogValidator(realmConfig), new ImplicitOnlyConnectionValidator())); Supplier catalogsApiSupplier = () -> { PolarisAdminService adminService =