diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 3cd12e1fb4c14..2632bc4ae7c8a 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -39,7 +39,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.freshstartree.codec.StarTreeCodec; +import org.opensearch.index.codec.startree.codec.StarTreeCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index 088838ad20c2a..fbc349bbade5e 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -38,8 +38,8 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.opensearch.common.lucene.Lucene; -import org.opensearch.index.codec.freshstartree.codec.StarTreeCodec; -import org.opensearch.index.codec.freshstartree.codec.StarTreeDocValuesFormat; +import org.opensearch.index.codec.startree.codec.StarTreeCodec; +import org.opensearch.index.codec.startree.codec.StarTreeDocValuesFormat; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java index 44915ab46cc44..8afabc3b6f660 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java index 8c528106d68c2..f679fd94d259f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Aggregated function type */ public enum AggregationFunctionType { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java index 97836196d83e6..93210310fd99f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Count value aggregator */ public class CountValueAggregator implements ValueAggregator { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java index 9ebc1d896a9fa..48779171ded7a 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Data type of doc values */ public enum DataType { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java index 3572643588648..9117a0d568273 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Sum value aggregator */ public class SumValueAggregator implements ValueAggregator { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java index a292aa7d7ba0b..d6b24928df8d6 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java index 4fe01e484bcc9..2988a7229a91c 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Value aggregator factory for a given aggregation type */ public class ValueAggregatorFactory { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java index d05d4fd5cb5cf..dd3be024439cf 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Aggregator classes for star tree */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java similarity index 79% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java index 81026ec0a0b5c..6d1fff81f649f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import java.time.temporal.ChronoField; import org.apache.logging.log4j.LogManager; @@ -22,29 +22,31 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.opensearch.common.time.DateUtils; -import org.opensearch.index.codec.freshstartree.aggregator.AggregationFunctionColumnPair; -import org.opensearch.index.codec.freshstartree.aggregator.AggregationFunctionType; -import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregator; -import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregatorFactory; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.node.StarTreeNode; -import org.opensearch.index.codec.freshstartree.util.BufferedAggregatedDocValues; +import org.opensearch.index.codec.startree.aggregator.AggregationFunctionColumnPair; +import org.opensearch.index.codec.startree.aggregator.AggregationFunctionType; +import org.opensearch.index.codec.startree.aggregator.ValueAggregator; +import org.opensearch.index.codec.startree.aggregator.ValueAggregatorFactory; +import org.opensearch.index.codec.startree.codec.SortedNumericDocValuesWriter; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.node.StarTreeNode; +import org.opensearch.index.codec.startree.util.BufferedAggregatedDocValues; import java.io.IOException; import java.util.ArrayList; @@ -97,12 +99,14 @@ public abstract class BaseSingleTreeBuilder { indexOutput = state.directory.createOutput(docFileName, state.context); CodecUtil.writeIndexHeader(indexOutput, "STARTreeCodec", 0, state.segmentInfo.getId(), state.segmentSuffix); dimensionsSplitOrder = new ArrayList<>(); + dimensionsSplitOrder.add("minute"); dimensionsSplitOrder.add("hour"); dimensionsSplitOrder.add("day"); dimensionsSplitOrder.add("month"); // dimensionsSplitOrder.add("year"); - dimensionsSplitOrder.add("status"); + dimensionsSplitOrder.add("elb_status"); + dimensionsSplitOrder.add("target_status"); _numDimensions = dimensionsSplitOrder.size(); _dimensionsSplitOrder = new String[_numDimensions]; _skipStarNodeCreationForDimensions = new HashSet<>(); @@ -111,10 +115,13 @@ public abstract class BaseSingleTreeBuilder { List functionColumnPairList = new ArrayList<>(); // TODO : pass function column pair - Remove hardcoding - functionColumnPairList.add("SUM__status"); + functionColumnPairList.add("SUM__elb_status"); + functionColumnPairList.add("SUM__target_status"); + functionColumnPairList.add("COUNT__target_status"); List aggregationSpecs = new ArrayList<>(); - aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__status")); - // aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__status")); + aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__elb_status")); + aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__target_status")); + //aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__elb_status")); _dimensionReaders = new SortedNumericDocValues[_numDimensions]; Set skipStarNodeCreationForDimensions = new HashSet<>(); @@ -147,7 +154,7 @@ public abstract class BaseSingleTreeBuilder { } // TODO : Removing hardcoding - _maxLeafRecords = 10000; // builderConfig.getMaxLeafRecords(); + _maxLeafRecords = 100; // builderConfig.getMaxLeafRecords(); } private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -253,12 +260,118 @@ public void build(Iterator recordIterator, boolean isMerge) throws IOExc logger.info("Finished creating aggregated documents, got aggregated records : {}", numAggregatedRecords); // Create doc values indices in disk - createDocValuesIndices(_docValuesConsumer); + createSortedDocValuesIndices(_docValuesConsumer); // Serialize and save in disk StarTreeBuilderUtils.serializeTree(indexOutput, _rootNode, _dimensionsSplitOrder, _numNodes); } + private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer) throws IOException { + List dimWriterList = new ArrayList<>(); + List metricWriterList = new ArrayList<>(); + FieldInfo[] dimFieldInfoArr = new FieldInfo[_dimensionReaders.length]; + FieldInfo[] metricFieldInfoArr = new FieldInfo[_metricReaders.length]; + int fieldNum = 0; + for (int i = 0; i < _dimensionReaders.length; i++) { + final FieldInfo fi = new FieldInfo( + _dimensionsSplitOrder[i] + "_dim", + fieldNum, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false + ); + dimFieldInfoArr[i] = fi; + final SortedNumericDocValuesWriter w = new SortedNumericDocValuesWriter(fi, Counter.newCounter()); + dimWriterList.add(w); + fieldNum++; + } + for (int i = 0; i < _metricReaders.length; i++) { + FieldInfo fi = new FieldInfo( + _metrics[i] + "_metric", + fieldNum, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false + ); + final SortedNumericDocValuesWriter w = new SortedNumericDocValuesWriter(fi, Counter.newCounter()); + metricWriterList.add(w); + metricFieldInfoArr[i] = fi; + fieldNum++; + } + + for (int docId = 0; docId < _numDocs; docId++) { + Record record = getStarTreeRecord(docId); + for (int i = 0; i < record._dimensions.length; i++) { + long val = record._dimensions[i]; + dimWriterList.get(i).addValue(docId, val); + } + for (int i = 0; i < record._metrics.length; i++) { + switch (_valueAggregators[i].getAggregatedValueType()) { + case LONG: + long val = (long) record._metrics[i]; + metricWriterList.get(i).addValue(docId, val); + break; + // TODO: support this + case DOUBLE: + // double doubleval = (double) record._metrics[i]; + // break; + case FLOAT: + case INT: + default: + throw new IllegalStateException("Unsupported value type"); + } + } + } + + for (int i = 0; i < _dimensionReaders.length; i++) { + final int finalI = i; + DocValuesProducer a1 = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + + return dimWriterList.get(finalI).getSortedNumericDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(dimFieldInfoArr[i], a1); + } + + for (int i = 0; i < _metricReaders.length; i++) { + final int finalI = i; + DocValuesProducer a1 = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + + return metricWriterList.get(finalI).getSortedNumericDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(metricFieldInfoArr[i], a1); + } + + + } + private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws IOException { PackedLongValues.Builder[] pendingDimArr = new PackedLongValues.Builder[_dimensionReaders.length]; PackedLongValues.Builder[] pendingMetricArr = new PackedLongValues.Builder[_metricReaders.length]; @@ -276,7 +389,7 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -299,7 +412,7 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -343,24 +456,24 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws final int finalI = i; DocValuesProducer a1 = new EmptyDocValuesProducer() { @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator()); + return DocValues.singleton(new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator())); } }; - docValuesConsumer.addNumericField(dimFieldInfoArr[i], a1); + docValuesConsumer.addSortedNumericField(dimFieldInfoArr[i], a1); } for (int i = 0; i < _metricReaders.length; i++) { final int finalI = i; DocValuesProducer a1 = new EmptyDocValuesProducer() { @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return new BufferedAggregatedDocValues(pendingMetricArr[finalI].build(), docsWithField.iterator()); + return DocValues.singleton(new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator())); } }; - docValuesConsumer.addNumericField(metricFieldInfoArr[i], a1); + docValuesConsumer.addSortedNumericField(metricFieldInfoArr[i], a1); } } @@ -560,7 +673,12 @@ Record getNextSegmentRecord() throws IOException { for (int i = 0; i < _numMetrics; i++) { // Ignore the column for COUNT aggregation function if (_metricReaders[i] != null) { - _metricReaders[i].nextDoc(); + try { + _metricReaders[i].nextDoc(); + } catch (Exception e) { + // TODO : handle null values in columns + logger.info(e); + } metrics[i] = _metricReaders[i].nextValue(); } } @@ -617,8 +735,13 @@ private long getTimeStampVal(final String fieldName, final long val) { long[] getNextSegmentRecordDimensions() throws IOException { long[] dimensions = new long[_numDimensions]; for (int i = 0; i < _numDimensions; i++) { - _dimensionReaders[i].nextDoc(); + try { + _dimensionReaders[i].nextDoc(); + } catch(Exception e) { + logger.info(e); + } dimensions[i] = getTimeStampVal(_dimensionsSplitOrder[i], _dimensionReaders[i].nextValue()); + //logger.info("Dim : {} , DimValue : {}", _dimensionsSplitOrder[i], _dimensionReaders[i].nextValue()); } return dimensions; } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java index c95778584bd97..84c963e8647b7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java @@ -14,13 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; @@ -28,8 +27,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; import org.apache.lucene.util.IOUtils; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.util.QuickSorter; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.util.QuickSorter; import java.io.IOException; import java.nio.ByteBuffer; @@ -129,12 +128,14 @@ private Iterator mergeRecords(List aggrList) t while (!endOfDoc) { long[] dims = new long[starTree.dimensionValues.size()]; int i = 0; - for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { - endOfDoc = dimValue.getValue().nextDoc() == DocIdSetIterator.NO_MORE_DOCS || dimValue.getValue().longValue() == -1; + for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { + int doc = dimValue.getValue().nextDoc(); + long val = dimValue.getValue().nextValue(); + + endOfDoc = doc == DocIdSetIterator.NO_MORE_DOCS || val == -1; if (endOfDoc) { break; } - long val = dimValue.getValue().longValue(); dims[i] = val; i++; } @@ -143,9 +144,9 @@ private Iterator mergeRecords(List aggrList) t } i = 0; Object[] metrics = new Object[starTree.metricValues.size()]; - for (Map.Entry metricValue : starTree.metricValues.entrySet()) { + for (Map.Entry metricValue : starTree.metricValues.entrySet()) { metricValue.getValue().nextDoc(); - metrics[i] = metricValue.getValue().longValue(); + metrics[i] = metricValue.getValue().nextValue(); i++; } Record record = new Record(dims, metrics); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java similarity index 91% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java index 6b193957342c9..f991f4d95f3ae 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import java.io.IOException; import java.util.ArrayList; @@ -58,12 +57,13 @@ private Iterator mergeRecords(List aggrList) t while (!endOfDoc) { long[] dims = new long[starTree.dimensionValues.size()]; int i = 0; - for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { - endOfDoc = dimValue.getValue().nextDoc() == DocIdSetIterator.NO_MORE_DOCS || dimValue.getValue().longValue() == -1; + for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { + int doc = dimValue.getValue().nextDoc(); + long val = dimValue.getValue().nextValue(); + endOfDoc = doc == DocIdSetIterator.NO_MORE_DOCS || val == -1; if (endOfDoc) { break; } - long val = dimValue.getValue().longValue(); dims[i] = val; i++; } @@ -72,9 +72,9 @@ private Iterator mergeRecords(List aggrList) t } i = 0; Object[] metrics = new Object[starTree.metricValues.size()]; - for (Map.Entry metricValue : starTree.metricValues.entrySet()) { + for (Map.Entry metricValue : starTree.metricValues.entrySet()) { metricValue.getValue().nextDoc(); - metrics[i] = metricValue.getValue().longValue(); + metrics[i] = metricValue.getValue().nextValue(); i++; } BaseSingleTreeBuilder.Record record = new BaseSingleTreeBuilder.Record(dims, metrics); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java index 43650d98c0871..8ff12372e93e7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; /** Class representing a star tree record */ public class Record { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java index f86468299d02f..4bd9e136ef322 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,7 +28,7 @@ import java.util.Queue; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.opensearch.index.codec.freshstartree.node.OffHeapStarTreeNode.SERIALIZABLE_SIZE_IN_BYTES; +import static org.opensearch.index.codec.startree.node.OffHeapStarTreeNode.SERIALIZABLE_SIZE_IN_BYTES; /** Util class for building star tree */ public class StarTreeBuilderUtils { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java index 298febc36487b..ccee510fdab6a 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Builder classes for star tree */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java index b02ceb8825597..6d4ab4ba91bce 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java index 623ca76b46743..f9f2ff799b4b8 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; @@ -161,8 +161,9 @@ public DocValuesProducer getMergeInstance() { public FieldInfo[] getFieldInfoArr() { List metrics = new ArrayList<>(); // TODO : remove this - metrics.add("status_sum"); - // metrics.add("status_count"); + metrics.add("elb_status_sum"); + metrics.add("target_status_sum"); + //metrics.add("elb_status_count"); FieldInfo[] fArr = new FieldInfo[dimensionSplitOrder.size() + metrics.size()]; int fieldNum = 0; for (int i = 0; i < dimensionSplitOrder.size(); i++) { @@ -173,7 +174,7 @@ public FieldInfo[] getFieldInfoArr() { false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -194,7 +195,7 @@ public FieldInfo[] getFieldInfoArr() { false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, diff --git a/server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java new file mode 100644 index 0000000000000..ece5963b17f82 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java @@ -0,0 +1,336 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.startree.codec; + +import java.io.IOException; +import java.util.Arrays; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.util.packed.PackedLongValues; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + + +public class SortedNumericDocValuesWriter { + private final PackedLongValues.Builder pending; // stream of all values + private PackedLongValues.Builder pendingCounts; // count of values per doc + private final DocsWithFieldSet docsWithField; + private final Counter iwBytesUsed; + private long bytesUsed; // this only tracks differences in 'pending' and 'pendingCounts' + private final FieldInfo fieldInfo; + private int currentDoc = -1; + private long[] currentValues = new long[8]; + private int currentUpto = 0; + + private PackedLongValues finalValues; + private PackedLongValues finalValuesCount; + + public SortedNumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { + this.fieldInfo = fieldInfo; + this.iwBytesUsed = iwBytesUsed; + pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); + docsWithField = new DocsWithFieldSet(); + bytesUsed = + pending.ramBytesUsed() + + docsWithField.ramBytesUsed() + + RamUsageEstimator.sizeOf(currentValues); + iwBytesUsed.addAndGet(bytesUsed); + } + + public void addValue(int docID, long value) { + assert docID >= currentDoc; + if (docID != currentDoc) { + finishCurrentDoc(); + currentDoc = docID; + } + + addOneValue(value); + updateBytesUsed(); + } + + // finalize currentDoc: this sorts the values in the current doc + private void finishCurrentDoc() { + if (currentDoc == -1) { + return; + } + if (currentUpto > 1) { + Arrays.sort(currentValues, 0, currentUpto); + } + for (int i = 0; i < currentUpto; i++) { + pending.add(currentValues[i]); + } + // record the number of values for this doc + if (pendingCounts != null) { + pendingCounts.add(currentUpto); + } else if (currentUpto != 1) { + pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); + for (int i = 0; i < docsWithField.cardinality(); ++i) { + pendingCounts.add(1); + } + pendingCounts.add(currentUpto); + } + currentUpto = 0; + + docsWithField.add(currentDoc); + } + + private void addOneValue(long value) { + if (currentUpto == currentValues.length) { + currentValues = ArrayUtil.grow(currentValues, currentValues.length + 1); + } + + currentValues[currentUpto] = value; + currentUpto++; + } + + private void updateBytesUsed() { + final long newBytesUsed = + pending.ramBytesUsed() + + (pendingCounts == null ? 0 : pendingCounts.ramBytesUsed()) + + docsWithField.ramBytesUsed() + + RamUsageEstimator.sizeOf(currentValues); + iwBytesUsed.addAndGet(newBytesUsed - bytesUsed); + bytesUsed = newBytesUsed; + } + + static final class LongValues { + final long[] offsets; + final PackedLongValues values; + + LongValues( + int maxDoc, + Sorter.DocMap sortMap, + SortedNumericDocValues oldValues, + float acceptableOverheadRatio) + throws IOException { + offsets = new long[maxDoc]; + PackedLongValues.Builder valuesBuiler = + PackedLongValues.packedBuilder(acceptableOverheadRatio); + int docID; + long offsetIndex = 1; // 0 means the doc has no values + while ((docID = oldValues.nextDoc()) != NO_MORE_DOCS) { + int newDocID = sortMap.oldToNew(docID); + int numValues = oldValues.docValueCount(); + valuesBuiler.add(numValues); + offsets[newDocID] = offsetIndex++; + for (int i = 0; i < numValues; i++) { + valuesBuiler.add(oldValues.nextValue()); + offsetIndex++; + } + } + values = valuesBuiler.build(); + } + } + + private SortedNumericDocValues getValues( + PackedLongValues values, PackedLongValues valueCounts, DocsWithFieldSet docsWithField) { + if (valueCounts == null) { + return DocValues.singleton(new BufferedNumericDocValues(values, docsWithField.iterator())); + } else { + return new BufferedSortedNumericDocValues(values, valueCounts, docsWithField.iterator()); + } + + } + + public SortedNumericDocValues getSortedNumericDocValues() + throws IOException { + final PackedLongValues values; + final PackedLongValues valueCounts; + if (finalValues == null) { + finishCurrentDoc(); + values = pending.build(); + valueCounts = pendingCounts == null ? null : pendingCounts.build(); + finalValues = values; + finalValuesCount = valueCounts; + } else { + values = finalValues; + valueCounts = finalValuesCount; + } + + final SortedNumericDocValues buf = getValues(values, valueCounts, docsWithField); + return buf; + } + + private static class BufferedSortedNumericDocValues extends SortedNumericDocValues { + final PackedLongValues.Iterator valuesIter; + final PackedLongValues.Iterator valueCountsIter; + final DocIdSetIterator docsWithField; + private int valueCount; + private int valueUpto; + + BufferedSortedNumericDocValues( + PackedLongValues values, PackedLongValues valueCounts, DocIdSetIterator docsWithField) { + valuesIter = values.iterator(); + valueCountsIter = valueCounts.iterator(); + this.docsWithField = docsWithField; + } + + @Override + public int docID() { + return docsWithField.docID(); + } + + @Override + public int nextDoc() throws IOException { + for (int i = valueUpto; i < valueCount; ++i) { + valuesIter.next(); + } + + int docID = docsWithField.nextDoc(); + if (docID != NO_MORE_DOCS) { + valueCount = Math.toIntExact(valueCountsIter.next()); + valueUpto = 0; + } + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int docValueCount() { + return valueCount; + } + + @Override + public long nextValue() { + valueUpto++; + return valuesIter.next(); + } + + @Override + public long cost() { + return docsWithField.cost(); + } + } + + static class SortingSortedNumericDocValues extends SortedNumericDocValues { + private final SortedNumericDocValues in; + private final LongValues values; + private int docID = -1; + private long upto; + private int numValues = -1; + + SortingSortedNumericDocValues(SortedNumericDocValues in, LongValues values) { + this.in = in; + this.values = values; + } + + @Override + public int docID() { + return docID; + } + + @Override + public int nextDoc() { + do { + docID++; + if (docID >= values.offsets.length) { + return docID = NO_MORE_DOCS; + } + } while (values.offsets[docID] <= 0); + upto = values.offsets[docID]; + numValues = Math.toIntExact(values.values.get(upto - 1)); + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException("use nextDoc instead"); + } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + upto = values.offsets[docID]; + if (values.offsets[docID] > 0) { + numValues = Math.toIntExact(values.values.get(upto - 1)); + return true; + } + return false; + } + + @Override + public long nextValue() { + return values.values.get(upto++); + } + + @Override + public long cost() { + return in.cost(); + } + + @Override + public int docValueCount() { + return numValues; + } + } + + public static class BufferedNumericDocValues extends NumericDocValues { + final PackedLongValues.Iterator iter; + final DocIdSetIterator docsWithField; + private long value; + + /** Values and doc with fields */ + public BufferedNumericDocValues(PackedLongValues values, DocIdSetIterator docsWithFields) { + this.iter = values.iterator(); + this.docsWithField = docsWithFields; + } + + @Override + public int docID() { + return docsWithField.docID(); + } + + @Override + public int nextDoc() throws IOException { + int docID = docsWithField.nextDoc(); + if (docID != NO_MORE_DOCS) { + value = iter.next(); + } + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return docsWithField.cost(); + } + + @Override + public long longValue() { + return value; + } + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java similarity index 74% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java index 2922e67255246..bf6583f8b379e 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; -import org.apache.lucene.index.NumericDocValues; -import org.opensearch.index.codec.freshstartree.node.StarTree; +import org.apache.lucene.index.SortedNumericDocValues; +import org.opensearch.index.codec.startree.node.StarTree; import java.util.Map; @@ -26,14 +26,14 @@ /** Star tree aggregated values holder for reader / query */ public class StarTreeAggregatedValues { public StarTree _starTree; - public Map dimensionValues; + public Map dimensionValues; - public Map metricValues; + public Map metricValues; public StarTreeAggregatedValues( StarTree starTree, - Map dimensionValues, - Map metricValues + Map dimensionValues, + Map metricValues ) { this._starTree = starTree; this.dimensionValues = dimensionValues; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java index e20deea192733..b73ee8b32c25e 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.CompoundFormat; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java index 7963cb4473d40..6e97cbc54eeba 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java similarity index 83% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java index d5139bf707199..2d19c80865451 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import java.util.LinkedHashMap; import org.apache.lucene.codecs.CodecUtil; @@ -28,8 +28,8 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.store.IndexInput; -import org.opensearch.index.codec.freshstartree.node.OffHeapStarTree; -import org.opensearch.index.codec.freshstartree.node.StarTree; +import org.opensearch.index.codec.startree.node.OffHeapStarTree; +import org.opensearch.index.codec.startree.node.StarTree; import java.io.IOException; import java.util.HashMap; @@ -47,9 +47,9 @@ public class StarTreeDocValuesReader extends DocValuesProducer { StarTree starTree; - Map dimensionValues; + Map dimensionValues; - Map metricValues; + Map metricValues; public static final String DATA_CODEC = "Lucene90DocValuesData"; public static final String META_CODEC = "Lucene90DocValuesMetadata"; @@ -73,11 +73,13 @@ public NumericDocValues getNumeric(FieldInfo field) throws IOException { public StarTreeAggregatedValues getAggregatedDocValues() throws IOException { List dimensionsSplitOrder = starTree.getDimensionNames(); for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getNumeric(dimensionsSplitOrder.get(i) + "_dim")); + dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getSortedNumeric(dimensionsSplitOrder.get(i) + "_dim")); } metricValues = new HashMap<>(); - metricValues.put("status_sum", valuesProducer.getNumeric("status_sum_metric")); - // metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); + metricValues.put("elb_status_sum", valuesProducer.getSortedNumeric("elb_status_sum_metric")); + metricValues.put("target_status_sum", valuesProducer.getSortedNumeric("target_status_sum_metric")); + //metricValues.put("elb_status_count", valuesProducer.getNumeric("elb_status_count_metric")); + //metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); return new StarTreeAggregatedValues(starTree, dimensionValues, metricValues); } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java index 2ccadf0422a9a..a4b7510501fd5 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; -import java.util.Collection; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,8 +26,8 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.codec.freshstartree.builder.BaseSingleTreeBuilder; -import org.opensearch.index.codec.freshstartree.builder.OffHeapBufferedSingleTreeBuilder; +import org.opensearch.index.codec.startree.builder.BaseSingleTreeBuilder; +import org.opensearch.index.codec.startree.builder.OffHeapBufferedSingleTreeBuilder; import java.io.IOException; import java.util.ArrayList; @@ -104,13 +103,10 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu dimensionReaders.put("day_dim", valuesProducer.getSortedNumeric(field)); dimensionReaders.put("month_dim", valuesProducer.getSortedNumeric(field)); // dimensionReaders.put("year_dim", valuesProducer.getSortedNumeric(field)); - } else { - // logger.info("Adding field : " + field.name); - dimensionReaders.put(field.name + "_dim", valuesProducer.getSortedNumeric(field)); - //dimensionsSplitOrder.add(field.name); } if (field.name.contains("status")) { // TODO : change this metric type + dimensionReaders.put(field.name + "_dim", valuesProducer.getSortedNumeric(field)); dimensionReaders.put(field.name + "_sum_metric", valuesProducer.getSortedNumeric(field)); } } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java index beea4beb3ddff..05f38a8fb5263 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Codec for performing aggregation during indexing */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java index 093c4503debf8..de386eb17c488 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java index 3a5b1b55ea9da..87bf130173919 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import org.apache.lucene.store.RandomAccessInput; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java index d1892cd564e15..80cdaefb5e3bb 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java index cf58d4904cdee..17996b233d1c4 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java index dbd8287650abe..0d81ff3c2ad49 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java @@ -16,4 +16,4 @@ */ /** node package */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java similarity index 80% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java index 3f754f9286b6e..0aa25aae60fc3 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java @@ -14,18 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.DocIdSetBuilder; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.node.StarTree; -import org.opensearch.index.codec.freshstartree.node.StarTreeNode; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.node.StarTree; +import org.opensearch.index.codec.startree.node.StarTreeNode; import java.io.IOException; import java.util.ArrayDeque; @@ -50,10 +48,15 @@ public class StarTreeFilter { static class StarTreeResult { final DocIdSetBuilder _matchedDocIds; final Set _remainingPredicateColumns; + final int numOfMatchedDocs; + final int maxMatchedDoc; - StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns) { + StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns, int numOfMatchedDocs, + int maxMatchedDoc) { _matchedDocIds = matchedDocIds; _remainingPredicateColumns = remainingPredicateColumns; + this.numOfMatchedDocs = numOfMatchedDocs; + this.maxMatchedDoc = maxMatchedDoc; } } @@ -63,9 +66,9 @@ static class StarTreeResult { private final Set _groupByColumns; DocIdSetBuilder docsWithField; - DocIdSetBuilder.BulkAdder adder; - Map dimValueMap; + DocIdSetBuilder.BulkAdder adder; + Map dimValueMap; public StarTreeFilter( StarTreeAggregatedValues starTreeAggrStructure, Map>> predicateEvaluators, @@ -93,27 +96,44 @@ public StarTreeFilter( // 1706268600 / (60*60*1000) * (60*60*1000) public DocIdSetIterator getStarTreeResult() throws IOException { StarTreeResult starTreeResult = traverseStarTree(); + //logger.info("Matched docs in star tree : {}" , starTreeResult.numOfMatchedDocs); List andIterators = new ArrayList<>(); andIterators.add(starTreeResult._matchedDocIds.build().iterator()); DocIdSetIterator docIdSetIterator = andIterators.get(0); + if(starTreeResult.maxMatchedDoc == -1) { + return docIdSetIterator; + } + int docCount = 0; for (String remainingPredicateColumn : starTreeResult._remainingPredicateColumns) { // TODO : set to max value of doc values - DocIdSetBuilder builder = new DocIdSetBuilder(Integer.MAX_VALUE); + DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.maxMatchedDoc); List> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn); - SortedNumericDocValues ndv = DocValues.singleton(this.dimValueMap.get(remainingPredicateColumn)); - + SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn); + List docIds = new ArrayList<>(); while (docIdSetIterator.nextDoc() != NO_MORE_DOCS) { + docCount++; int docID = docIdSetIterator.docID(); - ndv.advanceExact(docID); - long value = ndv.nextValue(); - for (Predicate compositePredicateEvaluator : compositePredicateEvaluators) { - // TODO : this might be expensive as its done against all doc values docs - if (compositePredicateEvaluator.test(value)) { - builder.grow(1).add(docID); - break; + if(ndv.advanceExact(docID)) { + final int valuesCount = ndv.docValueCount(); + long value = ndv.nextValue(); + for (Predicate compositePredicateEvaluator : compositePredicateEvaluators) { + // TODO : this might be expensive as its done against all doc values docs + if (compositePredicateEvaluator.test(value)) { + docIds.add(docID); + for (int i = 0; i < valuesCount - 1; i++) { + while(docIdSetIterator.nextDoc() != NO_MORE_DOCS) { + docIds.add(docIdSetIterator.docID()); + } + } + break; + } } } } + DocIdSetBuilder.BulkAdder adder = builder.grow(docIds.size()); + for(int docID : docIds) { + adder.add(docID); + } docIdSetIterator = builder.build().iterator(); } return docIdSetIterator; @@ -129,7 +149,7 @@ private StarTreeResult traverseStarTree() throws IOException { StarTree starTree = _starTree; List dimensionNames = starTree.getDimensionNames(); - logger.info("Dimension names {}", dimensionNames); + //logger.info("Dimension names {}", dimensionNames); StarTreeNode starTreeRootNode = starTree.getRoot(); // Track whether we have found a leaf node added to the queue. If we have found a leaf node, and @@ -149,7 +169,11 @@ private StarTreeResult traverseStarTree() throws IOException { globalRemainingPredicateColumns = new HashSet<>(remainingPredicateColumns); } + int matchedDocsCountInStarTree = 0; + int maxDocNum = -1; + StarTreeNode starTreeNode; + List docIds = new ArrayList<>(); while ((starTreeNode = queue.poll()) != null) { int dimensionId = starTreeNode.getDimensionId(); if (dimensionId > currentDimensionId) { @@ -165,8 +189,10 @@ private StarTreeResult traverseStarTree() throws IOException { // If all predicate columns and group-by columns are matched, we can use aggregated document if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) { - adder = docsWithField.grow(1); - adder.add(starTreeNode.getAggregatedDocId()); + int docId = starTreeNode.getAggregatedDocId(); + docIds.add(docId); + matchedDocsCountInStarTree++; + maxDocNum = docId > maxDocNum ? docId : maxDocNum; continue; } @@ -177,8 +203,9 @@ private StarTreeResult traverseStarTree() throws IOException { // remaining predicate columns for this node if (starTreeNode.isLeaf()) { for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { - adder = docsWithField.grow(1); - adder.add((int) i); + docIds.add((int)i); + matchedDocsCountInStarTree++; + maxDocNum = (int)i > maxDocNum ? (int)i : maxDocNum; } continue; } @@ -268,9 +295,15 @@ private StarTreeResult traverseStarTree() throws IOException { } } + adder = docsWithField.grow(docIds.size()); + for(int id : docIds) { + adder.add(id); + } return new StarTreeResult( docsWithField, - globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet() + globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(), + matchedDocsCountInStarTree, + maxDocNum ); } } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java index 6bef4545291ef..8a1025f59142c 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; @@ -27,7 +27,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Accountable; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java index 540d8a9e2e0bc..7f3f519be3227 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -18,7 +18,6 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.codec.freshstartree.builder.BaseSingleTreeBuilder; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; @@ -144,7 +143,7 @@ public static StarTreeQueryBuilder fromXContent(XContentParser parser) { protected Query doToQuery(QueryShardContext context) { // TODO : star tree supports either group by or filter if (predicateMap.size() > 0) { - logger.info("Predicates: {} ", this.groupBy.toString() ); + //logger.info("Predicates: {} ", this.groupBy.toString() ); return new StarTreeQuery(predicateMap, new HashSet<>()); } logger.info("Group by : {} ", this.groupBy.toString() ); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java index 8b04706698952..f019226645d73 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java @@ -16,4 +16,4 @@ */ /** query package - Todo : move it under main dir - keeping it here for readability */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java b/server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java index d3708a1ae8860..92b1d9edf7935 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.DocIdSetIterator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java b/server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java index 884a4401ade00..9a69963481a7f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java b/server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java index 2e5e33adc06b3..6a9b947204ec7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; /** copy of it.unimi.dsi.fastutil.Arrays.quickSort */ public class QuickSorter { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java b/server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java index 1df6a7e270698..cc1a4f3f8e721 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; /** Functional interface for swapper */ @FunctionalInterface diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java index 8ad0be6ce1848..4a62aea47bc1b 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java @@ -16,4 +16,4 @@ */ /** Util package todo: remove this, kept it temporarily for readibility */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 29950871b162b..1b26d079164b2 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -47,7 +47,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.codec.freshstartree.query.StarTreeQueryBuilder; +import org.opensearch.index.codec.startree.query.StarTreeQueryBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.BoostingQueryBuilder; import org.opensearch.index.query.CommonTermsQueryBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java index 368a91c09d876..b52e1f809422d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java @@ -29,6 +29,7 @@ public class StarTreeAggregationBuilder extends AbstractAggregationBuilder fieldCols; + private List metrics; public static final ObjectParser PARSER = ObjectParser.fromBuilder( NAME, StarTreeAggregationBuilder::new @@ -36,6 +37,7 @@ public class StarTreeAggregationBuilder extends AbstractAggregationBuilder strings) { @@ -43,6 +45,11 @@ private void groupby(List strings) { fieldCols.addAll(strings); } + private void metrics(List strings) { + metrics = new ArrayList<>(); + metrics.addAll(strings); + } + public StarTreeAggregationBuilder(String name) { super(name); } @@ -66,15 +73,20 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu public StarTreeAggregationBuilder(StreamInput in) throws IOException { super(in); String[] fieldArr = in.readOptionalStringArray(); + String[] metrics = in.readOptionalStringArray(); if (fieldArr != null) { fieldCols = Arrays.asList(fieldArr); } + if(metrics != null) { + this.metrics = Arrays.asList(metrics); + } } @Override protected void doWriteTo(StreamOutput out) throws IOException { // Nothing to write out.writeOptionalStringArray(fieldCols.toArray(new String[0])); + out.writeOptionalStringArray(metrics.toArray(new String[0])); } @Override @@ -88,7 +100,7 @@ protected AggregatorFactory doBuild( AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder ) throws IOException { - return new StarTreeAggregatorFactory(name, queryShardContext, parent, subFactoriesBuilder, metadata, fieldCols); + return new StarTreeAggregatorFactory(name, queryShardContext, parent, subFactoriesBuilder, metadata, fieldCols, metrics); } @Override diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index 941c75580dd42..cfcf2b95cbc47 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -8,10 +8,11 @@ package org.opensearch.search.aggregations.bucket.startree; +import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -20,7 +21,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; @@ -50,6 +51,8 @@ public class StarTreeAggregator extends BucketsAggregator implements SingleBucke private List fieldCols; + private List metrics; + final InternalStarTree.Factory starTreeFactory; private static final Logger logger = LogManager.getLogger(StarTreeAggregator.class); @@ -62,12 +65,14 @@ public StarTreeAggregator( SearchContext context, Aggregator parent, Map metadata, - List fieldCols + List fieldCols, + List metrics ) throws IOException { super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata); this._starTrees = starTrees; this.starTreeFactory = starTreeFactory; this.fieldCols = fieldCols; + this.metrics = metrics; } public static class StarTree implements Writeable, ToXContentObject { @@ -167,29 +172,34 @@ public InternalAggregation buildEmptyAggregation() { @Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { StarTreeAggregatedValues values = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues(); + final AtomicReference aggrVal = new AtomicReference<>(null); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - StarTreeAggregatedValues aggrVals = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues(); - - Map fieldColToDocValuesMap = new HashMap<>(); + if(aggrVal.get() == null) { + aggrVal.set((StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues()); + } + StarTreeAggregatedValues aggrVals = aggrVal.get(); + List fieldColToDocValuesMap = new ArrayList<>(); // TODO : validations for (String field : fieldCols) { - fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field)); + fieldColToDocValuesMap.add(aggrVals.dimensionValues.get(field)); } - NumericDocValues dv = aggrVals.metricValues.get("status_sum"); + // Another hardcoding + SortedNumericDocValues dv = aggrVals.metricValues.get(metrics.get(0)); if (dv.advanceExact(doc)) { - + long val1 = dv.nextValue(); String key = getKey(fieldColToDocValuesMap, doc); - + if(key.equals("") ) { + return; + } if (indexMap.containsKey(key)) { - sumMap.put(key, sumMap.getOrDefault(key, 0l) + dv.longValue()); + sumMap.put(key, sumMap.getOrDefault(key, 0l) + val1); } else { indexMap.put(key, indexMap.size()); - sumMap.put(key, dv.longValue()); + sumMap.put(key, dv.nextValue()); } - collectBucket(sub, doc, subBucketOrdinal(bucket, indexMap.get(key))); } } @@ -197,11 +207,11 @@ public void collect(int doc, long bucket) throws IOException { } - private String getKey(Map fieldColsMap, int doc) throws IOException { + private String getKey(List colsList, int doc) throws IOException { StringJoiner sj = new StringJoiner("-"); - for (Map.Entry fieldEntry : fieldColsMap.entrySet()) { - fieldEntry.getValue().advanceExact(doc); - long val = fieldEntry.getValue().longValue(); + for (SortedNumericDocValues col : colsList) { + col.advanceExact(doc); + long val = col.nextValue(); //System.out.println("Key field : " + fieldEntry.getKey() + " Value : " + val); sj.add("" + val); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java index 3f1f4693b597b..85e5a35696e5f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java @@ -21,6 +21,7 @@ public class StarTreeAggregatorFactory extends AggregatorFactory { private List fieldCols; + private List metrics; StarTreeAggregatorFactory( String name, @@ -28,10 +29,12 @@ public class StarTreeAggregatorFactory extends AggregatorFactory { AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metadata, - List fieldCols + List fieldCols, + List metrics ) throws IOException { super(name, queryShardContext, parent, subFactoriesBuilder, metadata); this.fieldCols = fieldCols; + this.metrics = metrics; } @Override @@ -41,7 +44,7 @@ public Aggregator createInternal( CardinalityUpperBound cardinality, Map metadata ) throws IOException { - return new StarTreeAggregator(name, factories, null, null, searchContext, parent, metadata, fieldCols); + return new StarTreeAggregator(name, factories, null, null, searchContext, parent, metadata, fieldCols, metrics); } @Override diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java index 4b8e882cd69bc..feca5627dcff0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.metrics; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ScoreMode; import org.opensearch.common.lease.Releasables; @@ -42,6 +44,7 @@ import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.search.aggregations.LeafBucketCollectorBase; +import org.opensearch.search.aggregations.bucket.startree.StarTreeAggregator; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; @@ -61,6 +64,8 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue { private DoubleArray sums; private DoubleArray compensations; + private static final Logger logger = LogManager.getLogger(SumAggregator.class); + SumAggregator( String name, @@ -95,11 +100,13 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { + //logger.info("collecting doc : {}", doc); sums = bigArrays.grow(sums, bucket + 1); compensations = bigArrays.grow(compensations, bucket + 1); if (values.advanceExact(doc)) { final int valuesCount = values.docValueCount(); + //logger.info("values count : {}" , valuesCount); // Compute the sum of double values with Kahan summation algorithm which is more // accurate than naive summation. double sum = sums.get(bucket); diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 25df72954122b..3057c566b4658 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -6,4 +6,4 @@ # compatible open source license. # -org.opensearch.index.codec.freshstartree.codec.StarTreeCodec +org.opensearch.index.codec.startree.codec.StarTreeCodec