Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions changelog.d/purge-task-session-tags.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
### Fixes

- The cleanup tasks scheduled by a drop with purge now vend STS credentials with the dropped table's
or view's catalog, namespace and table in the session tags and role session name
(`SESSION_TAGS_IN_SUBSCOPED_CREDENTIAL`, `SESSION_NAME_FIELDS_IN_SUBSCOPED_CREDENTIAL`).
Previously the task's name was used as the catalog and table, and the namespace was missing.

### Upgrade notes

- Cleanup tasks scheduled by a drop with purge before upgrading don't carry the catalog name, so
their STS credentials use `unknown` as the catalog session tag and leave the catalog out of the
role session name.
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,10 @@ public class PolarisTaskConstants {
* {@link #STORAGE_LOCATION} when present.
*/
public static final String STORAGE_LOCATIONS = "storageLocations";

/**
* Name of the catalog the dropped table or view belonged to, used for the credential vending
* context (e.g. STS session tags) of purge tasks.
*/
public static final String CATALOG_NAME = "catalogName";
}
Original file line number Diff line number Diff line change
Expand Up @@ -3193,12 +3193,20 @@ private void updateTableLike(
}
}

Map<String, String> cleanupProperties = storageProperties;
if (purge) {
// Cleanup tasks vend storage credentials outside of a catalog request, so they need the
// catalog name for the credential vending context (e.g. STS session tags).
cleanupProperties = new HashMap<>(storageProperties);
cleanupProperties.put(PolarisTaskConstants.CATALOG_NAME, catalogName);
}

return getMetaStoreManager()
.dropEntityIfExists(
getCurrentPolarisContext(),
PolarisEntity.toCoreList(catalogPath),
leafEntity,
storageProperties,
cleanupProperties,
purge);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.polaris.core.StructuredLogKeys;
import org.apache.polaris.core.auth.PolarisPrincipal;
Expand All @@ -36,6 +37,7 @@
import org.apache.polaris.core.entity.PolarisEntity;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.rest.NamespaceUtils;
import org.apache.polaris.core.storage.CredentialVendingContext;
import org.apache.polaris.core.storage.LocationGrant;
import org.apache.polaris.core.storage.PolarisStorageActions;
Expand All @@ -49,7 +51,9 @@
/**
* Request-scoped entry point for vending scoped storage credentials. Resolves the storage
* integration for the given entity path via {@link PolarisStorageIntegrationProvider}, builds a
* {@link CredentialVendingContext} from request-scoped state, and delegates to the integration.
* {@link CredentialVendingContext} from the resolved entity path and request-scoped state (or, for
* background tasks, from the catalog name and table identifier the task carries), and delegates to
* the integration.
*/
@RequestScoped
public class StorageAccessConfigProvider {
Expand Down Expand Up @@ -79,18 +83,75 @@ public StorageAccessConfig getStorageAccessConfig(
@NonNull Set<PolarisStorageActions> storageActions,
@NonNull Optional<String> refreshCredentialsEndpoint,
@NonNull PolarisResolvedPathWrapper resolvedPath) {
List<PolarisEntity> resolvedEntityPath = resolvedPath.getRawFullPath();
return getStorageAccessConfig(
tableIdentifier,
resolvedEntityPath,
tableLocations,
storageActions,
refreshCredentialsEndpoint,
buildCredentialVendingContext(resolvedEntityPath));
}

/**
* Vends storage credentials for a background task acting on {@code tableIdentifier}. The
* credential vending context is built from {@code catalogName} and {@code tableIdentifier};
* {@code resolvedTaskPath} is only used to look up the storage integration.
*/
public StorageAccessConfig getStorageAccessConfigForTask(
@NonNull Optional<String> catalogName,
@NonNull TableIdentifier tableIdentifier,
@NonNull Set<String> tableLocations,
@NonNull Set<PolarisStorageActions> storageActions,
@NonNull PolarisResolvedPathWrapper resolvedTaskPath) {
return getStorageAccessConfig(
tableIdentifier,
resolvedTaskPath.getRawFullPath(),
tableLocations,
storageActions,
Optional.empty(),
buildCredentialVendingContext(
catalogName, tableIdentifier.namespace(), Optional.of(tableIdentifier.name())));
}

private StorageAccessConfig getStorageAccessConfig(
@NonNull TableIdentifier tableIdentifier,
@NonNull List<PolarisEntity> resolvedEntityPath,
@NonNull Set<String> tableLocations,
@NonNull Set<PolarisStorageActions> storageActions,
@NonNull Optional<String> refreshCredentialsEndpoint,
@NonNull CredentialVendingContext credentialVendingContext) {
LOGGER
.atDebug()
.addKeyValue(StructuredLogKeys.TABLE_IDENTIFIER, tableIdentifier)
.addKeyValue(StructuredLogKeys.TABLE_LOCATION, tableLocations)
.log("Fetching client credentials for table");

StorageAccessConfig accessConfig =
getStorageAccessConfig(
resolvedPath.getRawFullPath(),
tableLocations,
storageActions,
refreshCredentialsEndpoint);
boolean skipCredentialSubscopingIndirection =
callContext
.getRealmConfig()
.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION);
PolarisStorageIntegration integration =
skipCredentialSubscopingIndirection
? null
: storageIntegrationProvider.getStorageIntegration(resolvedEntityPath);

StorageAccessConfig accessConfig;
if (integration == null) {
accessConfig = StorageAccessConfig.builder().supportsCredentialVending(false).build();
} else {
// Non-delegated loadTable still calls in here to fetch storage extra-properties
// (endpoint/region/path-style) and passes no actions; treat that as a READ grant so
// the integration emits a well-formed inline policy rather than calling STS empty-handed.
Set<PolarisStorageActions> effectiveActions =
storageActions.isEmpty() ? Set.of(PolarisStorageActions.READ) : storageActions;

accessConfig =
integration.getStorageAccessConfig(
List.of(new LocationGrant(tableLocations, effectiveActions)),
refreshCredentialsEndpoint,
credentialVendingContext);
}

LOGGER
.atDebug()
Expand All @@ -104,43 +165,40 @@ public StorageAccessConfig getStorageAccessConfig(
return accessConfig;
}

private StorageAccessConfig getStorageAccessConfig(
@NonNull List<PolarisEntity> resolvedEntityPath,
@NonNull Set<String> locations,
@NonNull Set<PolarisStorageActions> storageActions,
@NonNull Optional<String> refreshCredentialsEndpoint) {

boolean skipCredentialSubscopingIndirection =
callContext
.getRealmConfig()
.getConfig(FeatureConfiguration.SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION);
if (skipCredentialSubscopingIndirection) {
return StorageAccessConfig.builder().supportsCredentialVending(false).build();
}
private CredentialVendingContext buildCredentialVendingContext(
List<PolarisEntity> resolvedEntityPath) {
Optional<String> catalogName = Optional.empty();
Namespace namespace = Namespace.empty();
Optional<String> tableName = Optional.empty();

PolarisStorageIntegration integration =
storageIntegrationProvider.getStorageIntegration(resolvedEntityPath);
if (integration == null) {
return StorageAccessConfig.builder().supportsCredentialVending(false).build();
}
if (!resolvedEntityPath.isEmpty()) {
// First entity is the catalog; paths not rooted at a catalog leave it unset
PolarisEntity root = resolvedEntityPath.get(0);
if (root.getType() == PolarisEntityType.CATALOG) {
catalogName = Optional.of(root.getName());
}

CredentialVendingContext credentialVendingContext =
buildCredentialVendingContext(resolvedEntityPath);
// Last entity is the table/view
PolarisEntity leaf = resolvedEntityPath.get(resolvedEntityPath.size() - 1);
if (leaf.getType() == PolarisEntityType.TABLE_LIKE) {
tableName = Optional.of(leaf.getName());
}

// Non-delegated loadTable still calls in here to fetch storage extra-properties
// (endpoint/region/path-style) and passes no actions; treat that as a READ grant so
// the integration emits a well-formed inline policy rather than calling STS empty-handed.
Set<PolarisStorageActions> effectiveActions =
storageActions.isEmpty() ? Set.of(PolarisStorageActions.READ) : storageActions;
// Namespace entities are between catalog and leaf
if (resolvedEntityPath.size() > 2) {
namespace =
Namespace.of(
resolvedEntityPath.subList(1, resolvedEntityPath.size() - 1).stream()
.map(PolarisEntity::getName)
.toArray(String[]::new));
}
}

return integration.getStorageAccessConfig(
List.of(new LocationGrant(locations, effectiveActions)),
refreshCredentialsEndpoint,
credentialVendingContext);
return buildCredentialVendingContext(catalogName, namespace, tableName);
}

private CredentialVendingContext buildCredentialVendingContext(
List<PolarisEntity> resolvedEntityPath) {
Optional<String> catalogName, Namespace namespace, Optional<String> tableName) {
CredentialVendingContext.Builder builder = CredentialVendingContext.builder();

List<String> sessionTagFields =
Expand All @@ -149,27 +207,15 @@ private CredentialVendingContext buildCredentialVendingContext(
.getConfig(FeatureConfiguration.SESSION_TAGS_IN_SUBSCOPED_CREDENTIAL);

builder.realm(Optional.of(realmContext.getRealmIdentifier()));

if (!resolvedEntityPath.isEmpty()) {
// First entity is the catalog
builder.catalogName(Optional.of(resolvedEntityPath.get(0).getName()));

// Last entity is the table/view
PolarisEntity leaf = resolvedEntityPath.get(resolvedEntityPath.size() - 1);
if (leaf.getType() == PolarisEntityType.TABLE_LIKE
|| leaf.getType() == PolarisEntityType.TASK) {
builder.tableName(Optional.of(leaf.getName()));
}

// Namespace entities are between catalog and leaf
if (resolvedEntityPath.size() > 2) {
String namespace =
resolvedEntityPath.subList(1, resolvedEntityPath.size() - 1).stream()
.map(PolarisEntity::getName)
.collect(Collectors.joining("."));
builder.namespace(Optional.of(namespace));
}
}
builder.catalogName(catalogName);
// The single place where a namespace becomes the dot-separated string documented on
// CredentialVendingContext#namespace, so request and task paths stay consistent. Note that a
// level containing a dot is indistinguishable from nested levels in this value.
builder.namespace(
namespace.isEmpty()
? Optional.empty()
: Optional.of(NamespaceUtils.joinNamespace(namespace, ".")));
builder.tableName(tableName);

builder.principalName(Optional.of(polarisPrincipal.getName()));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.io.FileIO;
import org.apache.polaris.core.entity.PolarisTaskConstants;
import org.apache.polaris.core.entity.TaskEntity;
import org.apache.polaris.core.persistence.PolarisResolvedPathWrapper;
import org.apache.polaris.core.persistence.ResolvedPolarisEntity;
Expand Down Expand Up @@ -58,9 +59,11 @@ public FileIO apply(TaskEntity task, TableIdentifier identifier) {
new ResolvedPolarisEntity(task, List.of(), List.of());
PolarisResolvedPathWrapper resolvedPath =
new PolarisResolvedPathWrapper(List.of(resolvedTaskEntity));
Optional<String> catalogName =
Optional.ofNullable(properties.get(PolarisTaskConstants.CATALOG_NAME));
StorageAccessConfig storageAccessConfig =
accessConfigProvider.getStorageAccessConfig(
identifier, locations, storageActions, Optional.empty(), resolvedPath);
accessConfigProvider.getStorageAccessConfigForTask(
catalogName, identifier, locations, storageActions, resolvedPath);

String ioImpl =
properties.getOrDefault(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@
import org.apache.polaris.core.entity.PolarisEntityCore;
import org.apache.polaris.core.entity.PolarisEntitySubType;
import org.apache.polaris.core.entity.PolarisEntityType;
import org.apache.polaris.core.entity.PolarisTaskConstants;
import org.apache.polaris.core.entity.TaskEntity;
import org.apache.polaris.core.entity.table.IcebergTableLikeEntity;
import org.apache.polaris.core.exceptions.CommitConflictException;
Expand Down Expand Up @@ -2075,6 +2076,8 @@ public void testDropTableWithPurge() {
.getEntities();
Assertions.assertThat(tasks).hasSize(1);
TaskEntity taskEntity = TaskEntity.of(tasks.get(0));
Assertions.assertThat(taskEntity.getInternalPropertiesAsMap())
.containsEntry(PolarisTaskConstants.CATALOG_NAME, CATALOG_NAME);
var integration = storageIntegrationProvider.getStorageIntegration(List.of(taskEntity));
Map<String, String> credentials =
integration
Expand Down Expand Up @@ -2123,6 +2126,8 @@ public void testDropViewWithPurge() {
.getEntities()
.getFirst());
Map<String, String> properties = taskEntity.getInternalPropertiesAsMap();
Assertions.assertThat(properties)
.containsEntry(PolarisTaskConstants.CATALOG_NAME, CATALOG_NAME);
properties.put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO");
taskEntity =
TaskEntity.of(
Expand Down
Loading