Repository navigation
fix(cache): add configurable refresh buffer to StorageCredentialCache #5129
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
705a7fc
d41dcbf
814c1ba
c56c4bd
5b3cf60
348bec8
a2d3ab0
edd3715
b4d0ec4
429f2d0
3542f25
e414238
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| ### New Features | ||
|
|
||
| - Added the `STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS` feature flag (default `0`). When | ||
| non-zero, a cached storage credential is evicted this many seconds before it actually | ||
| expires, guaranteeing callers always receive credentials with at least this much validity | ||
| left; a freshly loaded credential that is already within the buffer is reloaded once rather | ||
| than handed back as-is, and if the reload is still under the buffer, an | ||
| `UnprocessableEntityException` is thrown rather than returning a credential that violates | ||
| the guarantee. Must be `>= 0` and less than `STORAGE_CREDENTIAL_DURATION_SECONDS`. When `0` | ||
| (the default), behavior is unchanged: credentials are kept for half of their remaining | ||
| lifetime. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -25,6 +25,7 @@ | |
| import java.time.Duration; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import org.apache.iceberg.exceptions.UnprocessableEntityException; | ||
| import org.apache.polaris.core.config.FeatureConfiguration; | ||
| import org.apache.polaris.core.config.RealmConfig; | ||
| import org.apache.polaris.core.storage.StorageAccessConfig; | ||
|
|
@@ -52,20 +53,22 @@ public StorageCredentialCache(StorageCredentialCacheConfig cacheConfig) { | |
| .expireAfter( | ||
| Expiry.creating( | ||
| (StorageCredentialCacheKey key, StorageCredentialCacheEntry entry) -> { | ||
| long remainingMs = entry.getExpirationTime() - System.currentTimeMillis(); | ||
| long bufferMs = entry.refreshBufferMs(); | ||
| long effectiveTtl = bufferMs > 0 ? remainingMs - bufferMs : remainingMs / 2; | ||
| long expireAfterMillis = | ||
| Math.max( | ||
| 0, | ||
| Math.min( | ||
| (entry.getExpirationTime() - System.currentTimeMillis()) / 2, | ||
| entry.maxCacheDurationMs())); | ||
| Math.max(0, Math.min(effectiveTtl, entry.maxCacheDurationMs())); | ||
| return Duration.ofMillis(expireAfterMillis); | ||
| })) | ||
| .build(key -> { | ||
| LOGGER.atDebug().log("StorageCredentialCache::load"); | ||
| StorageAccessConfig accessConfig = key.load(); | ||
| return new StorageCredentialCacheEntry( | ||
| accessConfig, maxCacheDurationMs(key.realmConfig())); | ||
| }); | ||
| .build( | ||
| key -> { | ||
| LOGGER.atDebug().log("StorageCredentialCache::load"); | ||
| StorageAccessConfig accessConfig = key.load(); | ||
| return new StorageCredentialCacheEntry( | ||
| accessConfig, | ||
| maxCacheDurationMs(key.realmConfig()), | ||
| refreshBufferMs(key.realmConfig())); | ||
| }); | ||
| } | ||
|
|
||
| /** How long credentials should remain in the cache. */ | ||
|
|
@@ -85,12 +88,76 @@ private long maxCacheDurationMs(RealmConfig realmConfig) { | |
| } | ||
| } | ||
|
|
||
| /** | ||
| * Minimum buffer to keep between credential expiry and cache eviction. | ||
| * | ||
| * <p>An out-of-range value is a misconfiguration, not something to fail a live credential | ||
| * request over: {@link org.apache.polaris.service.config.ProductionReadinessChecks} validates | ||
| * this same flag at server startup, where a bad value is surfaced loudly before any request can | ||
| * be affected. Here, at request time, it's too late for that -- so an invalid value is logged | ||
| * and ignored, falling back to the default half-lifetime behavior instead. | ||
| */ | ||
| private long refreshBufferMs(RealmConfig realmConfig) { | ||
| var refreshBufferSeconds = | ||
| realmConfig.getConfig(FeatureConfiguration.STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS); | ||
| var credentialDurationSeconds = | ||
| realmConfig.getConfig(FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS); | ||
| if (refreshBufferSeconds < 0 || refreshBufferSeconds >= credentialDurationSeconds) { | ||
| LOGGER | ||
| .atWarn() | ||
| .log( | ||
| "{} is {}, which is not >= 0 and less than {}={}. Falling back to the default " | ||
| + "half-lifetime behavior for this realm; fix the configured value.", | ||
| FeatureConfiguration.STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS.key(), | ||
| refreshBufferSeconds, | ||
| FeatureConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS.key(), | ||
| credentialDurationSeconds); | ||
| return 0L; | ||
| } | ||
| return refreshBufferSeconds * 1000L; | ||
| } | ||
|
|
||
| /** | ||
| * Return the cached {@link StorageAccessConfig} for {@code key}, loading it via {@link | ||
| * StorageCredentialCacheKey#load()} on miss. | ||
| * | ||
| * <p>The refresh buffer (when non-zero) is enforced on every call, not just on cache hits: a | ||
| * freshly loaded credential can itself already be within the buffer of expiring (e.g. a | ||
| * credential provider issuing a short-lived token), in which case Caffeine's {@code expireAfter} | ||
| * would give it a zero TTL but still hand it back on this same call. To honor the "clients always | ||
| * receive credentials with at least this much validity left" contract, such an entry is discarded | ||
| * and reloaded once, bypassing the stale copy. | ||
| * | ||
| * <p>If the reload is <em>still</em> under the buffer, the contract cannot be honored at all, so | ||
| * this throws rather than silently handing back an under-buffer credential -- a provider that | ||
| * can't clear its own configured buffer after a fresh fetch is a configuration problem the caller | ||
| * needs to see, not something to mask with a log line. It is not retried a third time: a provider | ||
| * issuing credentials shorter than the configured buffer will keep doing so, and hot-looping | ||
| * against it doesn't help. | ||
| */ | ||
| public StorageAccessConfig getOrLoad(StorageCredentialCacheKey key) { | ||
| return cache.get(key).toAccessConfig(); | ||
| StorageCredentialCacheEntry entry = cache.get(key); | ||
| if (isUnderRefreshBuffer(entry)) { | ||
| cache.invalidate(key); | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is Caffeine's native time-based expiry not robust enough?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i believe it controls how long something sits in the cache, and not about rejecting a value thats already bad the at the load time. thoughts? |
||
| entry = cache.get(key); | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If the provider keeps issuing tokens shorter than the buffer, this reload never helps. Both loads land inside the buffer, the entry gets a TTL of 0, and the next request does it all again. So that setup pays two provider calls plus a WARN on every request, with no caching. Can we either fail here, or return the first credential as best effort and skip the second load (maybe with a rate-limited warning)? 🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it. |
||
| if (isUnderRefreshBuffer(entry)) { | ||
| throw new UnprocessableEntityException( | ||
| "Storage credential provider returned a credential with only %d ms of validity left, " | ||
| + "which is still under the configured refresh buffer of %d ms, even after a " | ||
| + "reload. The credential provider may be issuing credentials shorter than " | ||
| + "STORAGE_CREDENTIAL_REFRESH_BUFFER_SECONDS.", | ||
| entry.getExpirationTime() - System.currentTimeMillis(), entry.refreshBufferMs()); | ||
| } | ||
| } | ||
| return entry.toAccessConfig(); | ||
| } | ||
|
|
||
| private static boolean isUnderRefreshBuffer(StorageCredentialCacheEntry entry) { | ||
| if (entry.refreshBufferMs() <= 0) { | ||
| return false; | ||
| } | ||
| long remainingMs = entry.getExpirationTime() - System.currentTimeMillis(); | ||
| return remainingMs < entry.refreshBufferMs(); | ||
| } | ||
|
|
||
| @VisibleForTesting | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Same as the CHANGELOG: "ensuring clients always receive" overpromises. Can we update it, and also mention that a positive buffer replaces the half-lifetime rule instead of combining with it? Then regenerate flags-polaris_features.md.
🤖 AI-generated review comment — agents make mistakes, so please verify before acting on it.