|
| 1 | +/* |
| 2 | + * Licensed to the Apache Software Foundation (ASF) under one |
| 3 | + * or more contributor license agreements. See the NOTICE file |
| 4 | + * distributed with this work for additional information |
| 5 | + * regarding copyright ownership. The ASF licenses this file |
| 6 | + * to you under the Apache License, Version 2.0 (the |
| 7 | + * "License"); you may not use this file except in compliance |
| 8 | + * with the License. You may obtain a copy of the License at |
| 9 | + * |
| 10 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, |
| 13 | + * software distributed under the License is distributed on an |
| 14 | + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 15 | + * KIND, either express or implied. See the License for the |
| 16 | + * specific language governing permissions and limitations |
| 17 | + * under the License. |
| 18 | + */ |
| 19 | + |
| 20 | +package org.apache.datafusion; |
| 21 | + |
| 22 | +import java.time.Duration; |
| 23 | + |
| 24 | +import org.apache.datafusion.protobuf.CacheManagerOptionsProto; |
| 25 | +import org.apache.datafusion.protobuf.ListFilesCacheOptionsProto; |
| 26 | + |
| 27 | +/** |
| 28 | + * Configuration for DataFusion's built-in {@code CacheManager}. Pass an instance to {@link |
| 29 | + * SessionContextBuilder#cacheManager(CacheManagerOptions)} to turn on any of the three |
| 30 | + * upstream-provided caches at session construction time. |
| 31 | + * |
| 32 | + * <p>The three caches are independent; calling one setter does not affect the others. A setter that |
| 33 | + * is never called leaves the upstream default in place — so existing callers that don't touch this |
| 34 | + * builder see no behavioral change. |
| 35 | + * |
| 36 | + * <ul> |
| 37 | + * <li>{@link Builder#fileMetadataCache(long)} — caches file-embedded metadata (e.g. parquet |
| 38 | + * footers / page metadata). Backed by {@code DefaultFilesMetadataCache} with the supplied |
| 39 | + * byte cap. |
| 40 | + * <li>{@link Builder#listFilesCache(long, Duration)} — caches results of object-store {@code |
| 41 | + * list} operations. Backed by {@code DefaultListFilesCache} with the supplied byte cap and |
| 42 | + * optional TTL ({@code null} = infinite). |
| 43 | + * <li>{@link Builder#fileStatisticsCache(boolean)} — caches per-file row counts and column |
| 44 | + * statistics. Backed by {@code DefaultFileStatisticsCache}; no bytewise cap exists upstream. |
| 45 | + * </ul> |
| 46 | + * |
| 47 | + * <p>This v1 surface configures the <em>built-in</em> cache implementations. Plugging custom Java |
| 48 | + * cache implementations through a JNI upcall path is intentionally out of scope — every cache |
| 49 | + * lookup is a hot path during scans, and routing them through Java would defeat the cache. |
| 50 | + */ |
| 51 | +public final class CacheManagerOptions { |
| 52 | + |
| 53 | + private final Long fileMetadataCacheMaxBytes; |
| 54 | + private final ListFilesCacheConfig listFilesCache; |
| 55 | + private final Boolean fileStatisticsCacheEnabled; |
| 56 | + |
| 57 | + private CacheManagerOptions(Builder b) { |
| 58 | + this.fileMetadataCacheMaxBytes = b.fileMetadataCacheMaxBytes; |
| 59 | + this.listFilesCache = b.listFilesCache; |
| 60 | + this.fileStatisticsCacheEnabled = b.fileStatisticsCacheEnabled; |
| 61 | + } |
| 62 | + |
| 63 | + /** Begin building a {@link CacheManagerOptions} instance. */ |
| 64 | + public static Builder builder() { |
| 65 | + return new Builder(); |
| 66 | + } |
| 67 | + |
| 68 | + CacheManagerOptionsProto toProto() { |
| 69 | + CacheManagerOptionsProto.Builder b = CacheManagerOptionsProto.newBuilder(); |
| 70 | + if (fileMetadataCacheMaxBytes != null) { |
| 71 | + b.setFileMetadataCacheMaxBytes(fileMetadataCacheMaxBytes); |
| 72 | + } |
| 73 | + if (listFilesCache != null) { |
| 74 | + ListFilesCacheOptionsProto.Builder lb = ListFilesCacheOptionsProto.newBuilder(); |
| 75 | + if (listFilesCache.maxBytes != null) { |
| 76 | + lb.setMaxBytes(listFilesCache.maxBytes); |
| 77 | + } |
| 78 | + if (listFilesCache.ttlMillis != null) { |
| 79 | + lb.setTtlMillis(listFilesCache.ttlMillis); |
| 80 | + } |
| 81 | + b.setListFilesCache(lb.build()); |
| 82 | + } |
| 83 | + if (fileStatisticsCacheEnabled != null) { |
| 84 | + b.setFileStatisticsCacheEnabled(fileStatisticsCacheEnabled); |
| 85 | + } |
| 86 | + return b.build(); |
| 87 | + } |
| 88 | + |
| 89 | + /** Internal carrier — `null`-vs-set is the source of truth for "user called this setter". */ |
| 90 | + private static final class ListFilesCacheConfig { |
| 91 | + final Long maxBytes; |
| 92 | + final Long ttlMillis; |
| 93 | + |
| 94 | + ListFilesCacheConfig(Long maxBytes, Long ttlMillis) { |
| 95 | + this.maxBytes = maxBytes; |
| 96 | + this.ttlMillis = ttlMillis; |
| 97 | + } |
| 98 | + } |
| 99 | + |
| 100 | + /** Builder for {@link CacheManagerOptions}. */ |
| 101 | + public static final class Builder { |
| 102 | + private Long fileMetadataCacheMaxBytes; |
| 103 | + private ListFilesCacheConfig listFilesCache; |
| 104 | + private Boolean fileStatisticsCacheEnabled; |
| 105 | + |
| 106 | + private Builder() {} |
| 107 | + |
| 108 | + /** |
| 109 | + * Enable the file-embedded metadata cache (parquet footers, page metadata) with the given byte |
| 110 | + * cap. The cap is the budget the upstream {@code DefaultFilesMetadataCache} uses to evict |
| 111 | + * entries; {@code 0} is legal and means "construct the cache but with a 0-byte budget" |
| 112 | + * (effectively disabled but observable in stats). |
| 113 | + * |
| 114 | + * @throws IllegalArgumentException if {@code maxBytes} is negative. |
| 115 | + */ |
| 116 | + public Builder fileMetadataCache(long maxBytes) { |
| 117 | + if (maxBytes < 0) { |
| 118 | + throw new IllegalArgumentException( |
| 119 | + "fileMetadataCache maxBytes must be non-negative, got " + maxBytes); |
| 120 | + } |
| 121 | + this.fileMetadataCacheMaxBytes = maxBytes; |
| 122 | + return this; |
| 123 | + } |
| 124 | + |
| 125 | + /** |
| 126 | + * Enable the list-files cache with the given byte cap and TTL. Pass {@code null} for {@code |
| 127 | + * ttl} to use upstream's "no expiration" semantics (entries are evicted only by capacity |
| 128 | + * pressure). |
| 129 | + * |
| 130 | + * @throws IllegalArgumentException if {@code maxBytes} is negative or {@code ttl} is negative. |
| 131 | + */ |
| 132 | + public Builder listFilesCache(long maxBytes, Duration ttl) { |
| 133 | + if (maxBytes < 0) { |
| 134 | + throw new IllegalArgumentException( |
| 135 | + "listFilesCache maxBytes must be non-negative, got " + maxBytes); |
| 136 | + } |
| 137 | + Long ttlMillis = null; |
| 138 | + if (ttl != null) { |
| 139 | + if (ttl.isNegative()) { |
| 140 | + throw new IllegalArgumentException("listFilesCache ttl must be non-negative, got " + ttl); |
| 141 | + } |
| 142 | + ttlMillis = ttl.toMillis(); |
| 143 | + } |
| 144 | + this.listFilesCache = new ListFilesCacheConfig(maxBytes, ttlMillis); |
| 145 | + return this; |
| 146 | + } |
| 147 | + |
| 148 | + /** |
| 149 | + * Enable the file-statistics cache (per-file row counts / column statistics). When {@code |
| 150 | + * enabled} is {@code true}, the upstream {@code DefaultFileStatisticsCache} is installed. When |
| 151 | + * {@code false}, the slot is explicitly set to disabled in the wire format — the same end-state |
| 152 | + * as never calling this setter, but distinguishable for testing. |
| 153 | + */ |
| 154 | + public Builder fileStatisticsCache(boolean enabled) { |
| 155 | + this.fileStatisticsCacheEnabled = enabled; |
| 156 | + return this; |
| 157 | + } |
| 158 | + |
| 159 | + public CacheManagerOptions build() { |
| 160 | + return new CacheManagerOptions(this); |
| 161 | + } |
| 162 | + } |
| 163 | +} |
0 commit comments