Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions changelog.d/storage-credential-refresh-buffer.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
### New Features

- Added the `STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS` feature flag (default `0`). When
non-zero, a cached storage credential is evicted this many seconds before it actually
expires, guaranteeing callers always receive credentials with at least this much validity
left; a freshly loaded credential that is already within the buffer is reloaded once rather
than handed back as-is, and if the reload is still under the buffer, an
`UnprocessableEntityException` is thrown rather than returning a credential that violates
the guarantee. Must be `>= 0` and less than `STORAGE_CREDENTIAL_DURATION_SECONDS`. When `0`
(the default), behavior is unchanged: credentials are kept for half of their remaining
lifetime.
Original file line number Diff line number Diff line change
Expand Up @@ -517,6 +517,22 @@ public static void enforceFeatureEnabledOrThrow(
.defaultValue(30 * 60) // 30 minutes
.buildFeatureConfiguration();

public static final FeatureConfiguration<Integer> STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS =
PolarisConfiguration.<Integer>builder()
.key("STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS")
.description(
"Minimum remaining validity (in seconds) that a cached storage credential must "
+ "have before it is considered stale and evicted. When non-zero, credentials "
+ "are removed from the cache this many seconds before they expire, ensuring "
+ "clients always receive credentials with at least this much validity left. "

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as the CHANGELOG: "ensuring clients always receive" overpromises. Can we update it, and also mention that a positive buffer replaces the half-lifetime rule instead of combining with it? Then regenerate flags-polaris_features.md.

🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it.

+ "When zero (the default), credentials are instead kept for half of their "
+ "remaining lifetime -- a reasonable choice on its own when the actual expiry "
+ "timeout can't be estimated upfront. Must be less than "
+ STORAGE_CREDENTIAL_DURATION_SECONDS.key()
+ ".")
.defaultValue(0)
.buildFeatureConfiguration();

public static final FeatureConfiguration<Integer> MAX_METADATA_REFRESH_RETRIES =
PolarisConfiguration.<Integer>builder()
.key("MAX_METADATA_REFRESH_RETRIES")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.time.Duration;
import java.util.Map;
import java.util.Optional;
import org.apache.iceberg.exceptions.UnprocessableEntityException;
import org.apache.polaris.core.config.FeatureConfiguration;
import org.apache.polaris.core.config.RealmConfig;
import org.apache.polaris.core.storage.StorageAccessConfig;
Expand Down Expand Up @@ -52,20 +53,22 @@ public StorageCredentialCache(StorageCredentialCacheConfig cacheConfig) {
.expireAfter(
Expiry.creating(
(StorageCredentialCacheKey key, StorageCredentialCacheEntry entry) -> {
long remainingMs = entry.getExpirationTime() - System.currentTimeMillis();
long bufferMs = entry.refreshBufferMs();
long effectiveTtl = bufferMs > 0 ? remainingMs - bufferMs : remainingMs / 2;
long expireAfterMillis =
Math.max(
0,
Math.min(
(entry.getExpirationTime() - System.currentTimeMillis()) / 2,
entry.maxCacheDurationMs()));
Math.max(0, Math.min(effectiveTtl, entry.maxCacheDurationMs()));
return Duration.ofMillis(expireAfterMillis);
}))
.build(key -> {
LOGGER.atDebug().log("StorageCredentialCache::load");
StorageAccessConfig accessConfig = key.load();
return new StorageCredentialCacheEntry(
accessConfig, maxCacheDurationMs(key.realmConfig()));
});
.build(
key -> {
LOGGER.atDebug().log("StorageCredentialCache::load");
StorageAccessConfig accessConfig = key.load();
return new StorageCredentialCacheEntry(
accessConfig,
maxCacheDurationMs(key.realmConfig()),
refreshBufferMs(key.realmConfig()));
});
}

/** How long credentials should remain in the cache. */
Expand All @@ -85,12 +88,76 @@ private long maxCacheDurationMs(RealmConfig realmConfig) {
}
}

/**
* Minimum buffer to keep between credential expiry and cache eviction.
*
* <p>An out-of-range value is a misconfiguration, not something to fail a live credential
* request over: {@link org.apache.polaris.service.config.ProductionReadinessChecks} validates
* this same flag at server startup, where a bad value is surfaced loudly before any request can
* be affected. Here, at request time, it's too late for that -- so an invalid value is logged
* and ignored, falling back to the default half-lifetime behavior instead.
*/
private long refreshBufferMs(RealmConfig realmConfig) {
var refreshBufferSeconds =
realmConfig.getConfig(FeatureConfiguration.STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS);
var credentialDurationSeconds =
realmConfig.getConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS);
if (refreshBufferSeconds < 0 || refreshBufferSeconds >= credentialDurationSeconds) {
LOGGER
.atWarn()
.log(
"{} is {}, which is not >= 0 and less than {}={}. Falling back to the default "
+ "half-lifetime behavior for this realm; fix the configured value.",
FeatureConfiguration.STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS.key(),
refreshBufferSeconds,
FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS.key(),
credentialDurationSeconds);
return 0L;
}
return refreshBufferSeconds * 1000L;
}

/**
* Return the cached {@link StorageAccessConfig} for {@code key}, loading it via {@link
* StorageCredentialCacheKey#load()} on miss.
*
* <p>The refresh buffer (when non-zero) is enforced on every call, not just on cache hits: a
* freshly loaded credential can itself already be within the buffer of expiring (e.g. a
* credential provider issuing a short-lived token), in which case Caffeine's {@code expireAfter}
* would give it a zero TTL but still hand it back on this same call. To honor the "clients always
* receive credentials with at least this much validity left" contract, such an entry is discarded
* and reloaded once, bypassing the stale copy.
*
* <p>If the reload is <em>still</em> under the buffer, the contract cannot be honored at all, so
* this throws rather than silently handing back an under-buffer credential -- a provider that
* can't clear its own configured buffer after a fresh fetch is a configuration problem the caller
* needs to see, not something to mask with a log line. It is not retried a third time: a provider
* issuing credentials shorter than the configured buffer will keep doing so, and hot-looping
* against it doesn't help.
*/
public StorageAccessConfig getOrLoad(StorageCredentialCacheKey key) {
return cache.get(key).toAccessConfig();
StorageCredentialCacheEntry entry = cache.get(key);
if (isUnderRefreshBuffer(entry)) {
cache.invalidate(key);

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is Caffeine's native time-based expiry not robust enough?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i believe it controls how long something sits in the cache, and not about rejecting a value thats already bad the at the load time. thoughts?

entry = cache.get(key);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the provider keeps issuing tokens shorter than the buffer, this reload never helps. Both loads land inside the buffer, the entry gets a TTL of 0, and the next request does it all again. So that setup pays two provider calls plus a WARN on every request, with no caching. Can we either fail here, or return the first credential as best effort and skip the second load (maybe with a rate-limited warning)?

🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it.

if (isUnderRefreshBuffer(entry)) {
throw new UnprocessableEntityException(
"Storage credential provider returned a credential with only %d ms of validity left, "
+ "which is still under the configured refresh buffer of %d ms, even after a "
+ "reload. The credential provider may be issuing credentials shorter than "
+ "STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS.",
entry.getExpirationTime() - System.currentTimeMillis(), entry.refreshBufferMs());
}
}
return entry.toAccessConfig();
}

private static boolean isUnderRefreshBuffer(StorageCredentialCacheEntry entry) {
if (entry.refreshBufferMs() <= 0) {
return false;
}
long remainingMs = entry.getExpirationTime() - System.currentTimeMillis();
return remainingMs < entry.refreshBufferMs();
}

@VisibleForTesting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,10 @@
* A storage credential cached entry.
*
* @param storageAccessConfig The scoped creds map that is fetched from a creds vending service
* @param refreshBufferMs Milliseconds before expiry at which the entry is considered stale
*/
public record StorageCredentialCacheEntry(
StorageAccessConfig storageAccessConfig, long maxCacheDurationMs) {
StorageAccessConfig storageAccessConfig, long maxCacheDurationMs, long refreshBufferMs) {

/** Get the expiration time in millisecond for the cached entry */
public long getExpirationTime() {
Expand Down
Loading
Loading