From add1ecf79c2162f44d1b7cde04f5c08aa2b2a020 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 12 Feb 2024 11:34:55 +0530 Subject: [PATCH 1/8] hardcoded values Signed-off-by: Bharathwaj G --- .../freshstartree/builder/BaseSingleTreeBuilder.java | 12 ++++++++---- .../codec/Lucene90DocValuesProducerCopy.java | 5 +++-- .../freshstartree/codec/StarTreeDocValuesReader.java | 6 ++++-- .../freshstartree/codec/StarTreeDocValuesWriter.java | 5 +---- .../bucket/startree/StarTreeAggregator.java | 3 ++- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java index 81026ec0a0b5c..b277a7a232f39 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java @@ -102,7 +102,8 @@ public abstract class BaseSingleTreeBuilder { dimensionsSplitOrder.add("day"); dimensionsSplitOrder.add("month"); // dimensionsSplitOrder.add("year"); - dimensionsSplitOrder.add("status"); + dimensionsSplitOrder.add("elb_status"); + dimensionsSplitOrder.add("target_status"); _numDimensions = dimensionsSplitOrder.size(); _dimensionsSplitOrder = new String[_numDimensions]; _skipStarNodeCreationForDimensions = new HashSet<>(); @@ -111,10 +112,13 @@ public abstract class BaseSingleTreeBuilder { List functionColumnPairList = new ArrayList<>(); // TODO : pass function column pair - Remove hardcoding - functionColumnPairList.add("SUM__status"); + functionColumnPairList.add("SUM__elb_status"); + functionColumnPairList.add("SUM__target_status"); + functionColumnPairList.add("COUNT__target_status"); List aggregationSpecs = new ArrayList<>(); - aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__status")); - // aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__status")); + aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__elb_status")); + aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__elb_status")); + aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__target_status")); _dimensionReaders = new SortedNumericDocValues[_numDimensions]; Set skipStarNodeCreationForDimensions = new HashSet<>(); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java index 623ca76b46743..b6facd1ec7861 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java @@ -161,8 +161,9 @@ public DocValuesProducer getMergeInstance() { public FieldInfo[] getFieldInfoArr() { List metrics = new ArrayList<>(); // TODO : remove this - metrics.add("status_sum"); - // metrics.add("status_count"); + metrics.add("elb_status_sum"); + metrics.add("elb_status_count"); + metrics.add("target_status_sum"); FieldInfo[] fArr = new FieldInfo[dimensionSplitOrder.size() + metrics.size()]; int fieldNum = 0; for (int i = 0; i < dimensionSplitOrder.size(); i++) { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java index d5139bf707199..2d4045bfebcb3 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java @@ -76,8 +76,10 @@ public StarTreeAggregatedValues getAggregatedDocValues() throws IOException { dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getNumeric(dimensionsSplitOrder.get(i) + "_dim")); } metricValues = new HashMap<>(); - metricValues.put("status_sum", valuesProducer.getNumeric("status_sum_metric")); - // metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); + metricValues.put("elb_status_sum", valuesProducer.getNumeric("elb_status_sum_metric")); + metricValues.put("elb_status_count", valuesProducer.getNumeric("elb_status_count_metric")); + metricValues.put("target_status_sum", valuesProducer.getNumeric("target_status_sum_metric")); + //metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); return new StarTreeAggregatedValues(starTree, dimensionValues, metricValues); } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java index 2ccadf0422a9a..cf393c146fcb3 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java @@ -104,13 +104,10 @@ public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProdu dimensionReaders.put("day_dim", valuesProducer.getSortedNumeric(field)); dimensionReaders.put("month_dim", valuesProducer.getSortedNumeric(field)); // dimensionReaders.put("year_dim", valuesProducer.getSortedNumeric(field)); - } else { - // logger.info("Adding field : " + field.name); - dimensionReaders.put(field.name + "_dim", valuesProducer.getSortedNumeric(field)); - //dimensionsSplitOrder.add(field.name); } if (field.name.contains("status")) { // TODO : change this metric type + dimensionReaders.put(field.name + "_dim", valuesProducer.getSortedNumeric(field)); dimensionReaders.put(field.name + "_sum_metric", valuesProducer.getSortedNumeric(field)); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index 941c75580dd42..93d0292ee62ed 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -178,7 +178,8 @@ public void collect(int doc, long bucket) throws IOException { for (String field : fieldCols) { fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field)); } - NumericDocValues dv = aggrVals.metricValues.get("status_sum"); + // Another hardcoding + NumericDocValues dv = aggrVals.metricValues.get("elb_status_count"); if (dv.advanceExact(doc)) { String key = getKey(fieldColToDocValuesMap, doc); From b9f124d3f90a5a98e7cddb62faede8f35c23901b Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Wed, 14 Feb 2024 21:29:33 +0530 Subject: [PATCH 2/8] fixes for elb Signed-off-by: Bharathwaj G --- .../builder/BaseSingleTreeBuilder.java | 16 +++++++++++++--- .../codec/Lucene90DocValuesProducerCopy.java | 2 +- .../codec/StarTreeDocValuesReader.java | 2 +- .../startree/StarTreeAggregationBuilder.java | 14 +++++++++++++- .../bucket/startree/StarTreeAggregator.java | 10 +++++++--- .../startree/StarTreeAggregatorFactory.java | 7 +++++-- 6 files changed, 40 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java index b277a7a232f39..5a85031226f75 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java @@ -117,8 +117,8 @@ public abstract class BaseSingleTreeBuilder { functionColumnPairList.add("COUNT__target_status"); List aggregationSpecs = new ArrayList<>(); aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__elb_status")); - aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__elb_status")); aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("SUM__target_status")); + //aggregationSpecs.add(AggregationFunctionColumnPair.fromColumnName("COUNT__elb_status")); _dimensionReaders = new SortedNumericDocValues[_numDimensions]; Set skipStarNodeCreationForDimensions = new HashSet<>(); @@ -564,7 +564,12 @@ Record getNextSegmentRecord() throws IOException { for (int i = 0; i < _numMetrics; i++) { // Ignore the column for COUNT aggregation function if (_metricReaders[i] != null) { - _metricReaders[i].nextDoc(); + try { + _metricReaders[i].nextDoc(); + } catch (Exception e) { + // TODO : handle null values in columns + logger.info(e); + } metrics[i] = _metricReaders[i].nextValue(); } } @@ -621,8 +626,13 @@ private long getTimeStampVal(final String fieldName, final long val) { long[] getNextSegmentRecordDimensions() throws IOException { long[] dimensions = new long[_numDimensions]; for (int i = 0; i < _numDimensions; i++) { - _dimensionReaders[i].nextDoc(); + try { + _dimensionReaders[i].nextDoc(); + } catch(Exception e) { + logger.info(e); + } dimensions[i] = getTimeStampVal(_dimensionsSplitOrder[i], _dimensionReaders[i].nextValue()); + //logger.info("Dim : {} , DimValue : {}", _dimensionsSplitOrder[i], _dimensionReaders[i].nextValue()); } return dimensions; } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java index b6facd1ec7861..190673fae7595 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java @@ -162,8 +162,8 @@ public FieldInfo[] getFieldInfoArr() { List metrics = new ArrayList<>(); // TODO : remove this metrics.add("elb_status_sum"); - metrics.add("elb_status_count"); metrics.add("target_status_sum"); + //metrics.add("elb_status_count"); FieldInfo[] fArr = new FieldInfo[dimensionSplitOrder.size() + metrics.size()]; int fieldNum = 0; for (int i = 0; i < dimensionSplitOrder.size(); i++) { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java index 2d4045bfebcb3..1c26151f30cdd 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java @@ -77,8 +77,8 @@ public StarTreeAggregatedValues getAggregatedDocValues() throws IOException { } metricValues = new HashMap<>(); metricValues.put("elb_status_sum", valuesProducer.getNumeric("elb_status_sum_metric")); - metricValues.put("elb_status_count", valuesProducer.getNumeric("elb_status_count_metric")); metricValues.put("target_status_sum", valuesProducer.getNumeric("target_status_sum_metric")); + //metricValues.put("elb_status_count", valuesProducer.getNumeric("elb_status_count_metric")); //metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); return new StarTreeAggregatedValues(starTree, dimensionValues, metricValues); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java index 368a91c09d876..b52e1f809422d 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregationBuilder.java @@ -29,6 +29,7 @@ public class StarTreeAggregationBuilder extends AbstractAggregationBuilder fieldCols; + private List metrics; public static final ObjectParser PARSER = ObjectParser.fromBuilder( NAME, StarTreeAggregationBuilder::new @@ -36,6 +37,7 @@ public class StarTreeAggregationBuilder extends AbstractAggregationBuilder strings) { @@ -43,6 +45,11 @@ private void groupby(List strings) { fieldCols.addAll(strings); } + private void metrics(List strings) { + metrics = new ArrayList<>(); + metrics.addAll(strings); + } + public StarTreeAggregationBuilder(String name) { super(name); } @@ -66,15 +73,20 @@ protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBu public StarTreeAggregationBuilder(StreamInput in) throws IOException { super(in); String[] fieldArr = in.readOptionalStringArray(); + String[] metrics = in.readOptionalStringArray(); if (fieldArr != null) { fieldCols = Arrays.asList(fieldArr); } + if(metrics != null) { + this.metrics = Arrays.asList(metrics); + } } @Override protected void doWriteTo(StreamOutput out) throws IOException { // Nothing to write out.writeOptionalStringArray(fieldCols.toArray(new String[0])); + out.writeOptionalStringArray(metrics.toArray(new String[0])); } @Override @@ -88,7 +100,7 @@ protected AggregatorFactory doBuild( AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder ) throws IOException { - return new StarTreeAggregatorFactory(name, queryShardContext, parent, subFactoriesBuilder, metadata, fieldCols); + return new StarTreeAggregatorFactory(name, queryShardContext, parent, subFactoriesBuilder, metadata, fieldCols, metrics); } @Override diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index 93d0292ee62ed..b910cd4a6f791 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -50,6 +50,8 @@ public class StarTreeAggregator extends BucketsAggregator implements SingleBucke private List fieldCols; + private List metrics; + final InternalStarTree.Factory starTreeFactory; private static final Logger logger = LogManager.getLogger(StarTreeAggregator.class); @@ -62,12 +64,14 @@ public StarTreeAggregator( SearchContext context, Aggregator parent, Map metadata, - List fieldCols + List fieldCols, + List metrics ) throws IOException { super(name, factories, context, parent, CardinalityUpperBound.MANY, metadata); this._starTrees = starTrees; this.starTreeFactory = starTreeFactory; this.fieldCols = fieldCols; + this.metrics = metrics; } public static class StarTree implements Writeable, ToXContentObject { @@ -179,7 +183,7 @@ public void collect(int doc, long bucket) throws IOException { fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field)); } // Another hardcoding - NumericDocValues dv = aggrVals.metricValues.get("elb_status_count"); + NumericDocValues dv = aggrVals.metricValues.get(metrics.get(0)); if (dv.advanceExact(doc)) { String key = getKey(fieldColToDocValuesMap, doc); @@ -190,7 +194,7 @@ public void collect(int doc, long bucket) throws IOException { indexMap.put(key, indexMap.size()); sumMap.put(key, dv.longValue()); } - + logger.info("Sum : {}" + sumMap.get(key)); collectBucket(sub, doc, subBucketOrdinal(bucket, indexMap.get(key))); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java index 3f1f4693b597b..85e5a35696e5f 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregatorFactory.java @@ -21,6 +21,7 @@ public class StarTreeAggregatorFactory extends AggregatorFactory { private List fieldCols; + private List metrics; StarTreeAggregatorFactory( String name, @@ -28,10 +29,12 @@ public class StarTreeAggregatorFactory extends AggregatorFactory { AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metadata, - List fieldCols + List fieldCols, + List metrics ) throws IOException { super(name, queryShardContext, parent, subFactoriesBuilder, metadata); this.fieldCols = fieldCols; + this.metrics = metrics; } @Override @@ -41,7 +44,7 @@ public Aggregator createInternal( CardinalityUpperBound cardinality, Map metadata ) throws IOException { - return new StarTreeAggregator(name, factories, null, null, searchContext, parent, metadata, fieldCols); + return new StarTreeAggregator(name, factories, null, null, searchContext, parent, metadata, fieldCols, metrics); } @Override From e48e60c953dee7cc934eb4b5807a33947ab45ca3 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Fri, 16 Feb 2024 14:34:25 +0530 Subject: [PATCH 3/8] disabling cache and adding loggers Signed-off-by: Bharathwaj G --- .../org/opensearch/index/IndexModule.java | 2 +- .../freshstartree/query/StarTreeFilter.java | 33 +++++++++++++++---- .../bucket/startree/StarTreeAggregator.java | 1 - 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index e29283724ebf8..a23e97e9b5ecf 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -267,7 +267,7 @@ public Iterator> settings() { // whether to use the query cache public static final Setting INDEX_QUERY_CACHE_ENABLED_SETTING = Setting.boolSetting( "index.queries.cache.enabled", - true, + false, Property.IndexScope ); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java index 3f754f9286b6e..2bef664ebff51 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java @@ -50,10 +50,12 @@ public class StarTreeFilter { static class StarTreeResult { final DocIdSetBuilder _matchedDocIds; final Set _remainingPredicateColumns; + final int numOfMatchedDocs; - StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns) { + StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns, int numOfMatchedDocs) { _matchedDocIds = matchedDocIds; _remainingPredicateColumns = remainingPredicateColumns; + this.numOfMatchedDocs = numOfMatchedDocs; } } @@ -63,9 +65,10 @@ static class StarTreeResult { private final Set _groupByColumns; DocIdSetBuilder docsWithField; + DocIdSetBuilder.BulkAdder adder; Map dimValueMap; - + int docNum; public StarTreeFilter( StarTreeAggregatedValues starTreeAggrStructure, Map>> predicateEvaluators, @@ -79,6 +82,7 @@ public StarTreeFilter( // TODO : this should be the maximum number of doc values docsWithField = new DocIdSetBuilder(Integer.MAX_VALUE); + docNum = 0; } /** @@ -92,20 +96,28 @@ public StarTreeFilter( // 1706268600 / (60*60*1000) * (60*60*1000) public DocIdSetIterator getStarTreeResult() throws IOException { + long startTime = System.nanoTime(); StarTreeResult starTreeResult = traverseStarTree(); + logger.info("Star tree traversal took : {}", System.nanoTime() - startTime); + startTime = System.nanoTime(); List andIterators = new ArrayList<>(); andIterators.add(starTreeResult._matchedDocIds.build().iterator()); DocIdSetIterator docIdSetIterator = andIterators.get(0); + int docCount = 0; for (String remainingPredicateColumn : starTreeResult._remainingPredicateColumns) { // TODO : set to max value of doc values - DocIdSetBuilder builder = new DocIdSetBuilder(Integer.MAX_VALUE); + DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs); List> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn); SortedNumericDocValues ndv = DocValues.singleton(this.dimValueMap.get(remainingPredicateColumn)); - + long ndvStartTime1 = System.nanoTime(); while (docIdSetIterator.nextDoc() != NO_MORE_DOCS) { + docCount++; + long ndvStartTime = System.nanoTime(); int docID = docIdSetIterator.docID(); ndv.advanceExact(docID); long value = ndv.nextValue(); + logger.info("Advancing took : {}", System.nanoTime() - ndvStartTime); + ndvStartTime = System.nanoTime(); for (Predicate compositePredicateEvaluator : compositePredicateEvaluators) { // TODO : this might be expensive as its done against all doc values docs if (compositePredicateEvaluator.test(value)) { @@ -113,9 +125,15 @@ public DocIdSetIterator getStarTreeResult() throws IOException { break; } } + logger.info("Predicate took : {}", System.nanoTime() - ndvStartTime); } + logger.info("Overall ndv took : {}", System.nanoTime() - ndvStartTime1); + long buildTime = System.nanoTime(); docIdSetIterator = builder.build().iterator(); + logger.info("Builder took : {}", System.nanoTime() - buildTime); } + logger.info("Doc value num : {}" , docCount); + logger.info("Rest of tree traversal took : {}", System.nanoTime() - startTime); return docIdSetIterator; } @@ -129,7 +147,7 @@ private StarTreeResult traverseStarTree() throws IOException { StarTree starTree = _starTree; List dimensionNames = starTree.getDimensionNames(); - logger.info("Dimension names {}", dimensionNames); + //logger.info("Dimension names {}", dimensionNames); StarTreeNode starTreeRootNode = starTree.getRoot(); // Track whether we have found a leaf node added to the queue. If we have found a leaf node, and @@ -166,6 +184,7 @@ private StarTreeResult traverseStarTree() throws IOException { // If all predicate columns and group-by columns are matched, we can use aggregated document if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) { adder = docsWithField.grow(1); + docNum++; adder.add(starTreeNode.getAggregatedDocId()); continue; } @@ -178,6 +197,7 @@ private StarTreeResult traverseStarTree() throws IOException { if (starTreeNode.isLeaf()) { for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { adder = docsWithField.grow(1); + docNum++; adder.add((int) i); } continue; @@ -270,7 +290,8 @@ private StarTreeResult traverseStarTree() throws IOException { return new StarTreeResult( docsWithField, - globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet() + globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(), + docNum ); } } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index b910cd4a6f791..ee836fc993ea4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -194,7 +194,6 @@ public void collect(int doc, long bucket) throws IOException { indexMap.put(key, indexMap.size()); sumMap.put(key, dv.longValue()); } - logger.info("Sum : {}" + sumMap.get(key)); collectBucket(sub, doc, subBucketOrdinal(bucket, indexMap.get(key))); } } From 197d659670c7c6991d57e008e73db5d76857cc2a Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Fri, 16 Feb 2024 21:22:38 +0530 Subject: [PATCH 4/8] Adding sorted numeric dv for dims and metrics Signed-off-by: Bharathwaj G --- .../builder/BaseSingleTreeBuilder.java | 130 ++++++- .../OffHeapBufferedSingleTreeBuilder.java | 12 +- .../builder/OnHeapSingleTreeBuilder.java | 11 +- .../codec/Lucene90DocValuesProducerCopy.java | 4 +- .../codec/SortedNumericDocValuesWriter.java | 340 ++++++++++++++++++ .../codec/StarTreeAggregatedValues.java | 9 +- .../codec/StarTreeDocValuesReader.java | 10 +- .../freshstartree/query/StarTreeFilter.java | 11 +- .../bucket/startree/StarTreeAggregator.java | 15 +- 9 files changed, 499 insertions(+), 43 deletions(-) create mode 100644 server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java index 5a85031226f75..7cf7106588c19 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java @@ -22,6 +22,7 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.EmptyDocValuesProducer; @@ -34,6 +35,8 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; @@ -42,6 +45,7 @@ import org.opensearch.index.codec.freshstartree.aggregator.AggregationFunctionType; import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregator; import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregatorFactory; +import org.opensearch.index.codec.freshstartree.codec.SortedNumericDocValuesWriter; import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; import org.opensearch.index.codec.freshstartree.node.StarTreeNode; import org.opensearch.index.codec.freshstartree.util.BufferedAggregatedDocValues; @@ -151,7 +155,7 @@ public abstract class BaseSingleTreeBuilder { } // TODO : Removing hardcoding - _maxLeafRecords = 10000; // builderConfig.getMaxLeafRecords(); + _maxLeafRecords = 1000; // builderConfig.getMaxLeafRecords(); } private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { @@ -257,12 +261,118 @@ public void build(Iterator recordIterator, boolean isMerge) throws IOExc logger.info("Finished creating aggregated documents, got aggregated records : {}", numAggregatedRecords); // Create doc values indices in disk - createDocValuesIndices(_docValuesConsumer); + createSortedDocValuesIndices(_docValuesConsumer); // Serialize and save in disk StarTreeBuilderUtils.serializeTree(indexOutput, _rootNode, _dimensionsSplitOrder, _numNodes); } + private void createSortedDocValuesIndices(DocValuesConsumer docValuesConsumer) throws IOException { + List dimWriterList = new ArrayList<>(); + List metricWriterList = new ArrayList<>(); + FieldInfo[] dimFieldInfoArr = new FieldInfo[_dimensionReaders.length]; + FieldInfo[] metricFieldInfoArr = new FieldInfo[_metricReaders.length]; + int fieldNum = 0; + for (int i = 0; i < _dimensionReaders.length; i++) { + final FieldInfo fi = new FieldInfo( + _dimensionsSplitOrder[i] + "_dim", + fieldNum, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false + ); + dimFieldInfoArr[i] = fi; + final SortedNumericDocValuesWriter w = new SortedNumericDocValuesWriter(fi, Counter.newCounter()); + dimWriterList.add(w); + fieldNum++; + } + for (int i = 0; i < _metricReaders.length; i++) { + FieldInfo fi = new FieldInfo( + _metrics[i] + "_metric", + fieldNum, + false, + false, + true, + IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, + DocValuesType.SORTED_NUMERIC, + -1, + Collections.emptyMap(), + 0, + 0, + 0, + 0, + VectorEncoding.FLOAT32, + VectorSimilarityFunction.EUCLIDEAN, + false + ); + final SortedNumericDocValuesWriter w = new SortedNumericDocValuesWriter(fi, Counter.newCounter()); + metricWriterList.add(w); + metricFieldInfoArr[i] = fi; + fieldNum++; + } + + for (int docId = 0; docId < _numDocs; docId++) { + Record record = getStarTreeRecord(docId); + for (int i = 0; i < record._dimensions.length; i++) { + long val = record._dimensions[i]; + dimWriterList.get(i).addValue(docId, val); + } + for (int i = 0; i < record._metrics.length; i++) { + switch (_valueAggregators[i].getAggregatedValueType()) { + case LONG: + long val = (long) record._metrics[i]; + metricWriterList.get(i).addValue(docId, val); + break; + // TODO: support this + case DOUBLE: + // double doubleval = (double) record._metrics[i]; + // break; + case FLOAT: + case INT: + default: + throw new IllegalStateException("Unsupported value type"); + } + } + } + + for (int i = 0; i < _dimensionReaders.length; i++) { + final int finalI = i; + DocValuesProducer a1 = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + + return dimWriterList.get(finalI).getSortedNumericDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(dimFieldInfoArr[i], a1); + } + + for (int i = 0; i < _metricReaders.length; i++) { + final int finalI = i; + DocValuesProducer a1 = new EmptyDocValuesProducer() { + @Override + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { + + return metricWriterList.get(finalI).getSortedNumericDocValues(); + } + }; + docValuesConsumer.addSortedNumericField(metricFieldInfoArr[i], a1); + } + + + } + private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws IOException { PackedLongValues.Builder[] pendingDimArr = new PackedLongValues.Builder[_dimensionReaders.length]; PackedLongValues.Builder[] pendingMetricArr = new PackedLongValues.Builder[_metricReaders.length]; @@ -280,7 +390,7 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -303,7 +413,7 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -347,24 +457,24 @@ private void createDocValuesIndices(DocValuesConsumer docValuesConsumer) throws final int finalI = i; DocValuesProducer a1 = new EmptyDocValuesProducer() { @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator()); + return DocValues.singleton(new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator())); } }; - docValuesConsumer.addNumericField(dimFieldInfoArr[i], a1); + docValuesConsumer.addSortedNumericField(dimFieldInfoArr[i], a1); } for (int i = 0; i < _metricReaders.length; i++) { final int finalI = i; DocValuesProducer a1 = new EmptyDocValuesProducer() { @Override - public NumericDocValues getNumeric(FieldInfo field) throws IOException { + public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException { - return new BufferedAggregatedDocValues(pendingMetricArr[finalI].build(), docsWithField.iterator()); + return DocValues.singleton(new BufferedAggregatedDocValues(pendingDimArr[finalI].build(), docsWithField.iterator())); } }; - docValuesConsumer.addNumericField(metricFieldInfoArr[i], a1); + docValuesConsumer.addSortedNumericField(metricFieldInfoArr[i], a1); } } diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java index c95778584bd97..af142ee7cbea8 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java @@ -129,12 +129,14 @@ private Iterator mergeRecords(List aggrList) t while (!endOfDoc) { long[] dims = new long[starTree.dimensionValues.size()]; int i = 0; - for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { - endOfDoc = dimValue.getValue().nextDoc() == DocIdSetIterator.NO_MORE_DOCS || dimValue.getValue().longValue() == -1; + for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { + int doc = dimValue.getValue().nextDoc(); + long val = dimValue.getValue().nextValue(); + + endOfDoc = doc == DocIdSetIterator.NO_MORE_DOCS || val == -1; if (endOfDoc) { break; } - long val = dimValue.getValue().longValue(); dims[i] = val; i++; } @@ -143,9 +145,9 @@ private Iterator mergeRecords(List aggrList) t } i = 0; Object[] metrics = new Object[starTree.metricValues.size()]; - for (Map.Entry metricValue : starTree.metricValues.entrySet()) { + for (Map.Entry metricValue : starTree.metricValues.entrySet()) { metricValue.getValue().nextDoc(); - metrics[i] = metricValue.getValue().longValue(); + metrics[i] = metricValue.getValue().nextValue(); i++; } Record record = new Record(dims, metrics); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java index 6b193957342c9..97bfdd5207816 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java @@ -58,12 +58,13 @@ private Iterator mergeRecords(List aggrList) t while (!endOfDoc) { long[] dims = new long[starTree.dimensionValues.size()]; int i = 0; - for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { - endOfDoc = dimValue.getValue().nextDoc() == DocIdSetIterator.NO_MORE_DOCS || dimValue.getValue().longValue() == -1; + for (Map.Entry dimValue : starTree.dimensionValues.entrySet()) { + int doc = dimValue.getValue().nextDoc(); + long val = dimValue.getValue().nextValue(); + endOfDoc = doc == DocIdSetIterator.NO_MORE_DOCS || val == -1; if (endOfDoc) { break; } - long val = dimValue.getValue().longValue(); dims[i] = val; i++; } @@ -72,9 +73,9 @@ private Iterator mergeRecords(List aggrList) t } i = 0; Object[] metrics = new Object[starTree.metricValues.size()]; - for (Map.Entry metricValue : starTree.metricValues.entrySet()) { + for (Map.Entry metricValue : starTree.metricValues.entrySet()) { metricValue.getValue().nextDoc(); - metrics[i] = metricValue.getValue().longValue(); + metrics[i] = metricValue.getValue().nextValue(); i++; } BaseSingleTreeBuilder.Record record = new BaseSingleTreeBuilder.Record(dims, metrics); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java index 190673fae7595..3a1eb8eafd533 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java @@ -174,7 +174,7 @@ public FieldInfo[] getFieldInfoArr() { false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, @@ -195,7 +195,7 @@ public FieldInfo[] getFieldInfoArr() { false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS, - DocValuesType.NUMERIC, + DocValuesType.SORTED_NUMERIC, -1, Collections.emptyMap(), 0, diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java new file mode 100644 index 0000000000000..ce2dd81125f46 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java @@ -0,0 +1,340 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.freshstartree.codec; + +import java.io.IOException; +import java.util.Arrays; +import org.apache.lucene.codecs.DocValuesConsumer; +import org.apache.lucene.codecs.DocValuesProducer; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.EmptyDocValuesProducer; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.util.packed.PackedLongValues; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + + +public class SortedNumericDocValuesWriter { + private final PackedLongValues.Builder pending; // stream of all values + private PackedLongValues.Builder pendingCounts; // count of values per doc + private final DocsWithFieldSet docsWithField; + private final Counter iwBytesUsed; + private long bytesUsed; // this only tracks differences in 'pending' and 'pendingCounts' + private final FieldInfo fieldInfo; + private int currentDoc = -1; + private long[] currentValues = new long[8]; + private int currentUpto = 0; + + private PackedLongValues finalValues; + private PackedLongValues finalValuesCount; + + public SortedNumericDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) { + this.fieldInfo = fieldInfo; + this.iwBytesUsed = iwBytesUsed; + pending = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); + docsWithField = new DocsWithFieldSet(); + bytesUsed = + pending.ramBytesUsed() + + docsWithField.ramBytesUsed() + + RamUsageEstimator.sizeOf(currentValues); + iwBytesUsed.addAndGet(bytesUsed); + } + + public void addValue(int docID, long value) { + assert docID >= currentDoc; + if (docID != currentDoc) { + finishCurrentDoc(); + currentDoc = docID; + } + + addOneValue(value); + updateBytesUsed(); + } + + // finalize currentDoc: this sorts the values in the current doc + private void finishCurrentDoc() { + if (currentDoc == -1) { + return; + } + if (currentUpto > 1) { + Arrays.sort(currentValues, 0, currentUpto); + } + for (int i = 0; i < currentUpto; i++) { + pending.add(currentValues[i]); + } + // record the number of values for this doc + if (pendingCounts != null) { + pendingCounts.add(currentUpto); + } else if (currentUpto != 1) { + pendingCounts = PackedLongValues.deltaPackedBuilder(PackedInts.COMPACT); + for (int i = 0; i < docsWithField.cardinality(); ++i) { + pendingCounts.add(1); + } + pendingCounts.add(currentUpto); + } + currentUpto = 0; + + docsWithField.add(currentDoc); + } + + private void addOneValue(long value) { + if (currentUpto == currentValues.length) { + currentValues = ArrayUtil.grow(currentValues, currentValues.length + 1); + } + + currentValues[currentUpto] = value; + currentUpto++; + } + + private void updateBytesUsed() { + final long newBytesUsed = + pending.ramBytesUsed() + + (pendingCounts == null ? 0 : pendingCounts.ramBytesUsed()) + + docsWithField.ramBytesUsed() + + RamUsageEstimator.sizeOf(currentValues); + iwBytesUsed.addAndGet(newBytesUsed - bytesUsed); + bytesUsed = newBytesUsed; + } + + static final class LongValues { + final long[] offsets; + final PackedLongValues values; + + LongValues( + int maxDoc, + Sorter.DocMap sortMap, + SortedNumericDocValues oldValues, + float acceptableOverheadRatio) + throws IOException { + offsets = new long[maxDoc]; + PackedLongValues.Builder valuesBuiler = + PackedLongValues.packedBuilder(acceptableOverheadRatio); + int docID; + long offsetIndex = 1; // 0 means the doc has no values + while ((docID = oldValues.nextDoc()) != NO_MORE_DOCS) { + int newDocID = sortMap.oldToNew(docID); + int numValues = oldValues.docValueCount(); + valuesBuiler.add(numValues); + offsets[newDocID] = offsetIndex++; + for (int i = 0; i < numValues; i++) { + valuesBuiler.add(oldValues.nextValue()); + offsetIndex++; + } + } + values = valuesBuiler.build(); + } + } + + private SortedNumericDocValues getValues( + PackedLongValues values, PackedLongValues valueCounts, DocsWithFieldSet docsWithField) { + if (valueCounts == null) { + return DocValues.singleton(new BufferedNumericDocValues(values, docsWithField.iterator())); + } else { + return new BufferedSortedNumericDocValues(values, valueCounts, docsWithField.iterator()); + } + + } + + public SortedNumericDocValues getSortedNumericDocValues() + throws IOException { + final PackedLongValues values; + final PackedLongValues valueCounts; + if (finalValues == null) { + finishCurrentDoc(); + values = pending.build(); + valueCounts = pendingCounts == null ? null : pendingCounts.build(); + finalValues = values; + finalValuesCount = valueCounts; + } else { + values = finalValues; + valueCounts = finalValuesCount; + } + + final SortedNumericDocValues buf = getValues(values, valueCounts, docsWithField); + return buf; + } + + private static class BufferedSortedNumericDocValues extends SortedNumericDocValues { + final PackedLongValues.Iterator valuesIter; + final PackedLongValues.Iterator valueCountsIter; + final DocIdSetIterator docsWithField; + private int valueCount; + private int valueUpto; + + BufferedSortedNumericDocValues( + PackedLongValues values, PackedLongValues valueCounts, DocIdSetIterator docsWithField) { + valuesIter = values.iterator(); + valueCountsIter = valueCounts.iterator(); + this.docsWithField = docsWithField; + } + + @Override + public int docID() { + return docsWithField.docID(); + } + + @Override + public int nextDoc() throws IOException { + for (int i = valueUpto; i < valueCount; ++i) { + valuesIter.next(); + } + + int docID = docsWithField.nextDoc(); + if (docID != NO_MORE_DOCS) { + valueCount = Math.toIntExact(valueCountsIter.next()); + valueUpto = 0; + } + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int docValueCount() { + return valueCount; + } + + @Override + public long nextValue() { + valueUpto++; + return valuesIter.next(); + } + + @Override + public long cost() { + return docsWithField.cost(); + } + } + + static class SortingSortedNumericDocValues extends SortedNumericDocValues { + private final SortedNumericDocValues in; + private final LongValues values; + private int docID = -1; + private long upto; + private int numValues = -1; + + SortingSortedNumericDocValues(SortedNumericDocValues in, LongValues values) { + this.in = in; + this.values = values; + } + + @Override + public int docID() { + return docID; + } + + @Override + public int nextDoc() { + do { + docID++; + if (docID >= values.offsets.length) { + return docID = NO_MORE_DOCS; + } + } while (values.offsets[docID] <= 0); + upto = values.offsets[docID]; + numValues = Math.toIntExact(values.values.get(upto - 1)); + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException("use nextDoc instead"); + } + + @Override + public boolean advanceExact(int target) throws IOException { + docID = target; + upto = values.offsets[docID]; + if (values.offsets[docID] > 0) { + numValues = Math.toIntExact(values.values.get(upto - 1)); + return true; + } + return false; + } + + @Override + public long nextValue() { + return values.values.get(upto++); + } + + @Override + public long cost() { + return in.cost(); + } + + @Override + public int docValueCount() { + return numValues; + } + } + + public static class BufferedNumericDocValues extends NumericDocValues { + final PackedLongValues.Iterator iter; + final DocIdSetIterator docsWithField; + private long value; + + /** Values and doc with fields */ + public BufferedNumericDocValues(PackedLongValues values, DocIdSetIterator docsWithFields) { + this.iter = values.iterator(); + this.docsWithField = docsWithFields; + } + + @Override + public int docID() { + return docsWithField.docID(); + } + + @Override + public int nextDoc() throws IOException { + int docID = docsWithField.nextDoc(); + if (docID != NO_MORE_DOCS) { + value = iter.next(); + } + return docID; + } + + @Override + public int advance(int target) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean advanceExact(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return docsWithField.cost(); + } + + @Override + public long longValue() { + return value; + } + } +} diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java index 2922e67255246..ec971b90ec624 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java @@ -17,6 +17,7 @@ package org.opensearch.index.codec.freshstartree.codec; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; import org.opensearch.index.codec.freshstartree.node.StarTree; import java.util.Map; @@ -26,14 +27,14 @@ /** Star tree aggregated values holder for reader / query */ public class StarTreeAggregatedValues { public StarTree _starTree; - public Map dimensionValues; + public Map dimensionValues; - public Map metricValues; + public Map metricValues; public StarTreeAggregatedValues( StarTree starTree, - Map dimensionValues, - Map metricValues + Map dimensionValues, + Map metricValues ) { this._starTree = starTree; this.dimensionValues = dimensionValues; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java index 1c26151f30cdd..8494188c1771e 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java @@ -47,9 +47,9 @@ public class StarTreeDocValuesReader extends DocValuesProducer { StarTree starTree; - Map dimensionValues; + Map dimensionValues; - Map metricValues; + Map metricValues; public static final String DATA_CODEC = "Lucene90DocValuesData"; public static final String META_CODEC = "Lucene90DocValuesMetadata"; @@ -73,11 +73,11 @@ public NumericDocValues getNumeric(FieldInfo field) throws IOException { public StarTreeAggregatedValues getAggregatedDocValues() throws IOException { List dimensionsSplitOrder = starTree.getDimensionNames(); for (int i = 0; i < dimensionsSplitOrder.size(); i++) { - dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getNumeric(dimensionsSplitOrder.get(i) + "_dim")); + dimensionValues.put(dimensionsSplitOrder.get(i), valuesProducer.getSortedNumeric(dimensionsSplitOrder.get(i) + "_dim")); } metricValues = new HashMap<>(); - metricValues.put("elb_status_sum", valuesProducer.getNumeric("elb_status_sum_metric")); - metricValues.put("target_status_sum", valuesProducer.getNumeric("target_status_sum_metric")); + metricValues.put("elb_status_sum", valuesProducer.getSortedNumeric("elb_status_sum_metric")); + metricValues.put("target_status_sum", valuesProducer.getSortedNumeric("target_status_sum_metric")); //metricValues.put("elb_status_count", valuesProducer.getNumeric("elb_status_count_metric")); //metricValues.put("status_count", valuesProducer.getNumeric("status_count_metric")); return new StarTreeAggregatedValues(starTree, dimensionValues, metricValues); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java index 2bef664ebff51..120b196ff8ef5 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java @@ -67,7 +67,7 @@ static class StarTreeResult { DocIdSetBuilder docsWithField; DocIdSetBuilder.BulkAdder adder; - Map dimValueMap; + Map dimValueMap; int docNum; public StarTreeFilter( StarTreeAggregatedValues starTreeAggrStructure, @@ -108,7 +108,7 @@ public DocIdSetIterator getStarTreeResult() throws IOException { // TODO : set to max value of doc values DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs); List> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn); - SortedNumericDocValues ndv = DocValues.singleton(this.dimValueMap.get(remainingPredicateColumn)); + SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn); long ndvStartTime1 = System.nanoTime(); while (docIdSetIterator.nextDoc() != NO_MORE_DOCS) { docCount++; @@ -184,8 +184,9 @@ private StarTreeResult traverseStarTree() throws IOException { // If all predicate columns and group-by columns are matched, we can use aggregated document if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) { adder = docsWithField.grow(1); - docNum++; - adder.add(starTreeNode.getAggregatedDocId()); + int docId = starTreeNode.getAggregatedDocId(); + adder.add(docId); + docNum = docId > docNum ? docId : docNum; continue; } @@ -197,8 +198,8 @@ private StarTreeResult traverseStarTree() throws IOException { if (starTreeNode.isLeaf()) { for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { adder = docsWithField.grow(1); - docNum++; adder.add((int) i); + docNum = (int)i > docNum ? (int)i : docNum; } continue; } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index ee836fc993ea4..9258c786200af 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.index.SortedNumericDocValues; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; @@ -176,23 +177,23 @@ protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucket public void collect(int doc, long bucket) throws IOException { StarTreeAggregatedValues aggrVals = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues(); - Map fieldColToDocValuesMap = new HashMap<>(); + Map fieldColToDocValuesMap = new HashMap<>(); // TODO : validations for (String field : fieldCols) { fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field)); } // Another hardcoding - NumericDocValues dv = aggrVals.metricValues.get(metrics.get(0)); + SortedNumericDocValues dv = aggrVals.metricValues.get(metrics.get(0)); if (dv.advanceExact(doc)) { String key = getKey(fieldColToDocValuesMap, doc); if (indexMap.containsKey(key)) { - sumMap.put(key, sumMap.getOrDefault(key, 0l) + dv.longValue()); + sumMap.put(key, sumMap.getOrDefault(key, 0l) + dv.nextValue()); } else { indexMap.put(key, indexMap.size()); - sumMap.put(key, dv.longValue()); + sumMap.put(key, dv.nextValue()); } collectBucket(sub, doc, subBucketOrdinal(bucket, indexMap.get(key))); } @@ -201,11 +202,11 @@ public void collect(int doc, long bucket) throws IOException { } - private String getKey(Map fieldColsMap, int doc) throws IOException { + private String getKey(Map fieldColsMap, int doc) throws IOException { StringJoiner sj = new StringJoiner("-"); - for (Map.Entry fieldEntry : fieldColsMap.entrySet()) { + for (Map.Entry fieldEntry : fieldColsMap.entrySet()) { fieldEntry.getValue().advanceExact(doc); - long val = fieldEntry.getValue().longValue(); + long val = fieldEntry.getValue().nextValue(); //System.out.println("Key field : " + fieldEntry.getKey() + " Value : " + val); sj.add("" + val); } From aab47d1ccf93673d421cbfd005bb71cc03544cbc Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Sat, 17 Feb 2024 09:42:48 +0530 Subject: [PATCH 5/8] Some perf improvement fixes Signed-off-by: Bharathwaj G --- .../builder/BaseSingleTreeBuilder.java | 2 +- .../freshstartree/query/StarTreeFilter.java | 50 ++++++++++--------- .../query/StarTreeQueryBuilder.java | 2 +- .../bucket/startree/StarTreeAggregator.java | 30 +++++++---- .../aggregations/metrics/SumAggregator.java | 7 +++ 5 files changed, 54 insertions(+), 37 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java index 7cf7106588c19..93b52fc583592 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java @@ -155,7 +155,7 @@ public abstract class BaseSingleTreeBuilder { } // TODO : Removing hardcoding - _maxLeafRecords = 1000; // builderConfig.getMaxLeafRecords(); + _maxLeafRecords = 100; // builderConfig.getMaxLeafRecords(); } private void constructStarTree(StarTreeBuilderUtils.TreeNode node, int startDocId, int endDocId) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java index 120b196ff8ef5..db0446717ddb9 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java @@ -96,10 +96,7 @@ public StarTreeFilter( // 1706268600 / (60*60*1000) * (60*60*1000) public DocIdSetIterator getStarTreeResult() throws IOException { - long startTime = System.nanoTime(); StarTreeResult starTreeResult = traverseStarTree(); - logger.info("Star tree traversal took : {}", System.nanoTime() - startTime); - startTime = System.nanoTime(); List andIterators = new ArrayList<>(); andIterators.add(starTreeResult._matchedDocIds.build().iterator()); DocIdSetIterator docIdSetIterator = andIterators.get(0); @@ -109,31 +106,33 @@ public DocIdSetIterator getStarTreeResult() throws IOException { DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs); List> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn); SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn); - long ndvStartTime1 = System.nanoTime(); + List docIds = new ArrayList<>(); while (docIdSetIterator.nextDoc() != NO_MORE_DOCS) { docCount++; - long ndvStartTime = System.nanoTime(); int docID = docIdSetIterator.docID(); - ndv.advanceExact(docID); - long value = ndv.nextValue(); - logger.info("Advancing took : {}", System.nanoTime() - ndvStartTime); - ndvStartTime = System.nanoTime(); - for (Predicate compositePredicateEvaluator : compositePredicateEvaluators) { - // TODO : this might be expensive as its done against all doc values docs - if (compositePredicateEvaluator.test(value)) { - builder.grow(1).add(docID); - break; + if(ndv.advanceExact(docID)) { + final int valuesCount = ndv.docValueCount(); + long value = ndv.nextValue(); + for (Predicate compositePredicateEvaluator : compositePredicateEvaluators) { + // TODO : this might be expensive as its done against all doc values docs + if (compositePredicateEvaluator.test(value)) { + docIds.add(docID); + for (int i = 0; i < valuesCount - 1; i++) { + while(docIdSetIterator.nextDoc() != NO_MORE_DOCS) { + docIds.add(docIdSetIterator.docID()); + } + } + break; + } } } - logger.info("Predicate took : {}", System.nanoTime() - ndvStartTime); } - logger.info("Overall ndv took : {}", System.nanoTime() - ndvStartTime1); - long buildTime = System.nanoTime(); + DocIdSetBuilder.BulkAdder adder = builder.grow(docIds.size()); + for(int docID : docIds) { + adder.add(docID); + } docIdSetIterator = builder.build().iterator(); - logger.info("Builder took : {}", System.nanoTime() - buildTime); } - logger.info("Doc value num : {}" , docCount); - logger.info("Rest of tree traversal took : {}", System.nanoTime() - startTime); return docIdSetIterator; } @@ -168,6 +167,7 @@ private StarTreeResult traverseStarTree() throws IOException { } StarTreeNode starTreeNode; + List docIds = new ArrayList<>(); while ((starTreeNode = queue.poll()) != null) { int dimensionId = starTreeNode.getDimensionId(); if (dimensionId > currentDimensionId) { @@ -183,9 +183,8 @@ private StarTreeResult traverseStarTree() throws IOException { // If all predicate columns and group-by columns are matched, we can use aggregated document if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) { - adder = docsWithField.grow(1); int docId = starTreeNode.getAggregatedDocId(); - adder.add(docId); + docIds.add(docId); docNum = docId > docNum ? docId : docNum; continue; } @@ -197,8 +196,7 @@ private StarTreeResult traverseStarTree() throws IOException { // remaining predicate columns for this node if (starTreeNode.isLeaf()) { for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { - adder = docsWithField.grow(1); - adder.add((int) i); + docIds.add((int)i); docNum = (int)i > docNum ? (int)i : docNum; } continue; @@ -289,6 +287,10 @@ private StarTreeResult traverseStarTree() throws IOException { } } + adder = docsWithField.grow(docIds.size()); + for(int id : docIds) { + adder.add(id); + } return new StarTreeResult( docsWithField, globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(), diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java index 540d8a9e2e0bc..fc2c019557d6f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java @@ -144,7 +144,7 @@ public static StarTreeQueryBuilder fromXContent(XContentParser parser) { protected Query doToQuery(QueryShardContext context) { // TODO : star tree supports either group by or filter if (predicateMap.size() > 0) { - logger.info("Predicates: {} ", this.groupBy.toString() ); + //logger.info("Predicates: {} ", this.groupBy.toString() ); return new StarTreeQuery(predicateMap, new HashSet<>()); } logger.info("Group by : {} ", this.groupBy.toString() ); diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index 9258c786200af..f2fd5a45799e2 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -8,6 +8,9 @@ package org.opensearch.search.aggregations.bucket.startree; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; @@ -172,25 +175,30 @@ public InternalAggregation buildEmptyAggregation() { @Override protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { StarTreeAggregatedValues values = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues(); + final AtomicReference aggrVal = new AtomicReference<>(null); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { - StarTreeAggregatedValues aggrVals = (StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues(); - - Map fieldColToDocValuesMap = new HashMap<>(); + if(aggrVal.get() == null) { + aggrVal.set((StarTreeAggregatedValues) ctx.reader().getAggregatedDocValues()); + } + StarTreeAggregatedValues aggrVals = aggrVal.get(); + List fieldColToDocValuesMap = new ArrayList<>(); // TODO : validations for (String field : fieldCols) { - fieldColToDocValuesMap.put(field, aggrVals.dimensionValues.get(field)); + fieldColToDocValuesMap.add(aggrVals.dimensionValues.get(field)); } // Another hardcoding SortedNumericDocValues dv = aggrVals.metricValues.get(metrics.get(0)); if (dv.advanceExact(doc)) { - + long val1 = dv.nextValue(); String key = getKey(fieldColToDocValuesMap, doc); - + if(key.equals("") ) { + return; + } if (indexMap.containsKey(key)) { - sumMap.put(key, sumMap.getOrDefault(key, 0l) + dv.nextValue()); + sumMap.put(key, sumMap.getOrDefault(key, 0l) + val1); } else { indexMap.put(key, indexMap.size()); sumMap.put(key, dv.nextValue()); @@ -202,11 +210,11 @@ public void collect(int doc, long bucket) throws IOException { } - private String getKey(Map fieldColsMap, int doc) throws IOException { + private String getKey(List colsList, int doc) throws IOException { StringJoiner sj = new StringJoiner("-"); - for (Map.Entry fieldEntry : fieldColsMap.entrySet()) { - fieldEntry.getValue().advanceExact(doc); - long val = fieldEntry.getValue().nextValue(); + for (SortedNumericDocValues col : colsList) { + col.advanceExact(doc); + long val = col.nextValue(); //System.out.println("Key field : " + fieldEntry.getKey() + " Value : " + val); sj.add("" + val); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java index 4b8e882cd69bc..feca5627dcff0 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/metrics/SumAggregator.java @@ -31,6 +31,8 @@ package org.opensearch.search.aggregations.metrics; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ScoreMode; import org.opensearch.common.lease.Releasables; @@ -42,6 +44,7 @@ import org.opensearch.search.aggregations.InternalAggregation; import org.opensearch.search.aggregations.LeafBucketCollector; import org.opensearch.search.aggregations.LeafBucketCollectorBase; +import org.opensearch.search.aggregations.bucket.startree.StarTreeAggregator; import org.opensearch.search.aggregations.support.ValuesSource; import org.opensearch.search.aggregations.support.ValuesSourceConfig; import org.opensearch.search.internal.SearchContext; @@ -61,6 +64,8 @@ public class SumAggregator extends NumericMetricsAggregator.SingleValue { private DoubleArray sums; private DoubleArray compensations; + private static final Logger logger = LogManager.getLogger(SumAggregator.class); + SumAggregator( String name, @@ -95,11 +100,13 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { + //logger.info("collecting doc : {}", doc); sums = bigArrays.grow(sums, bucket + 1); compensations = bigArrays.grow(compensations, bucket + 1); if (values.advanceExact(doc)) { final int valuesCount = values.docValueCount(); + //logger.info("values count : {}" , valuesCount); // Compute the sum of double values with Kahan summation algorithm which is more // accurate than naive summation. double sum = sums.get(bucket); From ebb9da1d1f9477175a783a79ddf37d3c1f0c745e Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Sat, 17 Feb 2024 15:43:48 +0530 Subject: [PATCH 6/8] enabling cache and 10k leaf Signed-off-by: Bharathwaj G --- .../org/opensearch/index/IndexModule.java | 2 +- .../builder/BaseSingleTreeBuilder.java | 1 + .../freshstartree/query/StarTreeFilter.java | 24 ++++++++++++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java index a23e97e9b5ecf..e29283724ebf8 100644 --- a/server/src/main/java/org/opensearch/index/IndexModule.java +++ b/server/src/main/java/org/opensearch/index/IndexModule.java @@ -267,7 +267,7 @@ public Iterator> settings() { // whether to use the query cache public static final Setting INDEX_QUERY_CACHE_ENABLED_SETTING = Setting.boolSetting( "index.queries.cache.enabled", - false, + true, Property.IndexScope ); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java index 93b52fc583592..e62a69ccf803a 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java @@ -101,6 +101,7 @@ public abstract class BaseSingleTreeBuilder { indexOutput = state.directory.createOutput(docFileName, state.context); CodecUtil.writeIndexHeader(indexOutput, "STARTreeCodec", 0, state.segmentInfo.getId(), state.segmentSuffix); dimensionsSplitOrder = new ArrayList<>(); + dimensionsSplitOrder.add("minute"); dimensionsSplitOrder.add("hour"); dimensionsSplitOrder.add("day"); diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java index db0446717ddb9..10219249e3bed 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java @@ -18,8 +18,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.DocIdSetBuilder; @@ -51,11 +49,14 @@ static class StarTreeResult { final DocIdSetBuilder _matchedDocIds; final Set _remainingPredicateColumns; final int numOfMatchedDocs; + final int maxMatchedDoc; - StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns, int numOfMatchedDocs) { + StarTreeResult(DocIdSetBuilder matchedDocIds, Set remainingPredicateColumns, int numOfMatchedDocs, + int maxMatchedDoc) { _matchedDocIds = matchedDocIds; _remainingPredicateColumns = remainingPredicateColumns; this.numOfMatchedDocs = numOfMatchedDocs; + this.maxMatchedDoc = maxMatchedDoc; } } @@ -68,7 +69,6 @@ static class StarTreeResult { DocIdSetBuilder.BulkAdder adder; Map dimValueMap; - int docNum; public StarTreeFilter( StarTreeAggregatedValues starTreeAggrStructure, Map>> predicateEvaluators, @@ -82,7 +82,6 @@ public StarTreeFilter( // TODO : this should be the maximum number of doc values docsWithField = new DocIdSetBuilder(Integer.MAX_VALUE); - docNum = 0; } /** @@ -97,13 +96,14 @@ public StarTreeFilter( // 1706268600 / (60*60*1000) * (60*60*1000) public DocIdSetIterator getStarTreeResult() throws IOException { StarTreeResult starTreeResult = traverseStarTree(); + logger.info("Matched docs in star tree : {}" , starTreeResult.numOfMatchedDocs); List andIterators = new ArrayList<>(); andIterators.add(starTreeResult._matchedDocIds.build().iterator()); DocIdSetIterator docIdSetIterator = andIterators.get(0); int docCount = 0; for (String remainingPredicateColumn : starTreeResult._remainingPredicateColumns) { // TODO : set to max value of doc values - DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.numOfMatchedDocs); + DocIdSetBuilder builder = new DocIdSetBuilder(starTreeResult.maxMatchedDoc); List> compositePredicateEvaluators = _predicateEvaluators.get(remainingPredicateColumn); SortedNumericDocValues ndv = this.dimValueMap.get(remainingPredicateColumn); List docIds = new ArrayList<>(); @@ -166,6 +166,9 @@ private StarTreeResult traverseStarTree() throws IOException { globalRemainingPredicateColumns = new HashSet<>(remainingPredicateColumns); } + int matchedDocsCountInStarTree = 0; + int maxDocNum = -1; + StarTreeNode starTreeNode; List docIds = new ArrayList<>(); while ((starTreeNode = queue.poll()) != null) { @@ -185,7 +188,8 @@ private StarTreeResult traverseStarTree() throws IOException { if (remainingPredicateColumns.isEmpty() && remainingGroupByColumns.isEmpty()) { int docId = starTreeNode.getAggregatedDocId(); docIds.add(docId); - docNum = docId > docNum ? docId : docNum; + matchedDocsCountInStarTree++; + maxDocNum = docId > maxDocNum ? docId : maxDocNum; continue; } @@ -197,7 +201,8 @@ private StarTreeResult traverseStarTree() throws IOException { if (starTreeNode.isLeaf()) { for (long i = starTreeNode.getStartDocId(); i < starTreeNode.getEndDocId(); i++) { docIds.add((int)i); - docNum = (int)i > docNum ? (int)i : docNum; + matchedDocsCountInStarTree++; + maxDocNum = (int)i > maxDocNum ? (int)i : maxDocNum; } continue; } @@ -294,7 +299,8 @@ private StarTreeResult traverseStarTree() throws IOException { return new StarTreeResult( docsWithField, globalRemainingPredicateColumns != null ? globalRemainingPredicateColumns : Collections.emptySet(), - docNum + matchedDocsCountInStarTree, + maxDocNum ); } } From e4397f3a248fd3284affad4358d827cf575961d9 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Tue, 27 Feb 2024 18:48:12 +0530 Subject: [PATCH 7/8] fix for no match in star tree Signed-off-by: Bharathwaj G --- .../index/codec/freshstartree/query/StarTreeFilter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java index 10219249e3bed..71b6b4947321d 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java @@ -96,10 +96,13 @@ public StarTreeFilter( // 1706268600 / (60*60*1000) * (60*60*1000) public DocIdSetIterator getStarTreeResult() throws IOException { StarTreeResult starTreeResult = traverseStarTree(); - logger.info("Matched docs in star tree : {}" , starTreeResult.numOfMatchedDocs); + //logger.info("Matched docs in star tree : {}" , starTreeResult.numOfMatchedDocs); List andIterators = new ArrayList<>(); andIterators.add(starTreeResult._matchedDocIds.build().iterator()); DocIdSetIterator docIdSetIterator = andIterators.get(0); + if(starTreeResult.maxMatchedDoc == -1) { + return docIdSetIterator; + } int docCount = 0; for (String remainingPredicateColumn : starTreeResult._remainingPredicateColumns) { // TODO : set to max value of doc values From e99cefe908a27b3649d71fa64eff397c398edc13 Mon Sep 17 00:00:00 2001 From: Bharathwaj G Date: Mon, 8 Apr 2024 14:25:39 +0530 Subject: [PATCH 8/8] Package rename Signed-off-by: Bharathwaj G --- .../opensearch/index/codec/CodecService.java | 2 +- .../PerFieldMappingPostingFormatCodec.java | 4 ++-- .../AggregationFunctionColumnPair.java | 2 +- .../aggregator/AggregationFunctionType.java | 2 +- .../aggregator/CountValueAggregator.java | 2 +- .../aggregator/DataType.java | 2 +- .../aggregator/SumValueAggregator.java | 2 +- .../aggregator/ValueAggregator.java | 2 +- .../aggregator/ValueAggregatorFactory.java | 2 +- .../aggregator/package-info.java | 2 +- .../builder/BaseSingleTreeBuilder.java | 20 +++++++++---------- .../OffHeapBufferedSingleTreeBuilder.java | 7 +++---- .../builder/OnHeapSingleTreeBuilder.java | 5 ++--- .../builder/Record.java | 2 +- .../builder/StarTreeBuilderUtils.java | 4 ++-- .../builder/package-info.java | 2 +- .../codec/Lucene90DocValuesConsumerCopy.java | 2 +- .../codec/Lucene90DocValuesProducerCopy.java | 2 +- .../codec/SortedNumericDocValuesWriter.java | 6 +----- .../codec/StarTreeAggregatedValues.java | 5 ++--- .../codec/StarTreeCodec.java | 2 +- .../codec/StarTreeDocValuesFormat.java | 2 +- .../codec/StarTreeDocValuesReader.java | 6 +++--- .../codec/StarTreeDocValuesWriter.java | 7 +++---- .../codec/package-info.java | 2 +- .../node/OffHeapStarTree.java | 2 +- .../node/OffHeapStarTreeNode.java | 2 +- .../node/StarTree.java | 2 +- .../node/StarTreeNode.java | 2 +- .../node/package-info.java | 2 +- .../query/StarTreeFilter.java | 8 ++++---- .../query/StarTreeQuery.java | 4 ++-- .../query/StarTreeQueryBuilder.java | 3 +-- .../query/package-info.java | 2 +- .../util/BufferedAggregatedDocValues.java | 2 +- .../util/IntComparator.java | 2 +- .../util/QuickSorter.java | 2 +- .../util/Swapper.java | 2 +- .../util/package-info.java | 2 +- .../org/opensearch/search/SearchModule.java | 2 +- .../bucket/startree/StarTreeAggregator.java | 5 +---- .../services/org.apache.lucene.codecs.Codec | 2 +- 42 files changed, 64 insertions(+), 78 deletions(-) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/AggregationFunctionColumnPair.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/AggregationFunctionType.java (95%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/CountValueAggregator.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/DataType.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/SumValueAggregator.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/ValueAggregator.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/ValueAggregatorFactory.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/aggregator/package-info.java (93%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/BaseSingleTreeBuilder.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/OffHeapBufferedSingleTreeBuilder.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/OnHeapSingleTreeBuilder.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/Record.java (96%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/StarTreeBuilderUtils.java (96%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/builder/package-info.java (93%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/Lucene90DocValuesConsumerCopy.java (99%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/Lucene90DocValuesProducerCopy.java (99%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/SortedNumericDocValuesWriter.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/StarTreeAggregatedValues.java (90%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/StarTreeCodec.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/StarTreeDocValuesFormat.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/StarTreeDocValuesReader.java (95%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/StarTreeDocValuesWriter.java (96%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/codec/package-info.java (93%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/node/OffHeapStarTree.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/node/OffHeapStarTreeNode.java (99%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/node/StarTree.java (95%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/node/StarTreeNode.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/node/package-info.java (93%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/query/StarTreeFilter.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/query/StarTreeQuery.java (95%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/query/StarTreeQueryBuilder.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/query/package-info.java (94%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/util/BufferedAggregatedDocValues.java (97%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/util/IntComparator.java (95%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/util/QuickSorter.java (98%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/util/Swapper.java (94%) rename server/src/main/java/org/opensearch/index/codec/{freshstartree => startree}/util/package-info.java (94%) diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index 3cd12e1fb4c14..2632bc4ae7c8a 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -39,7 +39,7 @@ import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.IndexSettings; -import org.opensearch.index.codec.freshstartree.codec.StarTreeCodec; +import org.opensearch.index.codec.startree.codec.StarTreeCodec; import org.opensearch.index.mapper.MapperService; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index 088838ad20c2a..fbc349bbade5e 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -38,8 +38,8 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene95.Lucene95Codec; import org.opensearch.common.lucene.Lucene; -import org.opensearch.index.codec.freshstartree.codec.StarTreeCodec; -import org.opensearch.index.codec.freshstartree.codec.StarTreeDocValuesFormat; +import org.opensearch.index.codec.startree.codec.StarTreeCodec; +import org.opensearch.index.codec.startree.codec.StarTreeDocValuesFormat; import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java index 44915ab46cc44..8afabc3b6f660 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionColumnPair.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionColumnPair.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java index 8c528106d68c2..f679fd94d259f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/AggregationFunctionType.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/AggregationFunctionType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Aggregated function type */ public enum AggregationFunctionType { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java index 97836196d83e6..93210310fd99f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/CountValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/CountValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Count value aggregator */ public class CountValueAggregator implements ValueAggregator { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java index 9ebc1d896a9fa..48779171ded7a 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/DataType.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/DataType.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Data type of doc values */ public enum DataType { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java index 3572643588648..9117a0d568273 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/SumValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/SumValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Sum value aggregator */ public class SumValueAggregator implements ValueAggregator { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java index a292aa7d7ba0b..d6b24928df8d6 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** * A value aggregator that pre-aggregates on the input values for a specific type of aggregation. diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java index 4fe01e484bcc9..2988a7229a91c 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/ValueAggregatorFactory.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/ValueAggregatorFactory.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; /** Value aggregator factory for a given aggregation type */ public class ValueAggregatorFactory { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java index d05d4fd5cb5cf..dd3be024439cf 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/aggregator/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/aggregator/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Aggregator classes for star tree */ -package org.opensearch.index.codec.freshstartree.aggregator; +package org.opensearch.index.codec.startree.aggregator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java index e62a69ccf803a..6d1fff81f649f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/BaseSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/BaseSingleTreeBuilder.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import java.time.temporal.ChronoField; import org.apache.logging.log4j.LogManager; @@ -29,26 +29,24 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; import org.opensearch.common.time.DateUtils; -import org.opensearch.index.codec.freshstartree.aggregator.AggregationFunctionColumnPair; -import org.opensearch.index.codec.freshstartree.aggregator.AggregationFunctionType; -import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregator; -import org.opensearch.index.codec.freshstartree.aggregator.ValueAggregatorFactory; -import org.opensearch.index.codec.freshstartree.codec.SortedNumericDocValuesWriter; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.node.StarTreeNode; -import org.opensearch.index.codec.freshstartree.util.BufferedAggregatedDocValues; +import org.opensearch.index.codec.startree.aggregator.AggregationFunctionColumnPair; +import org.opensearch.index.codec.startree.aggregator.AggregationFunctionType; +import org.opensearch.index.codec.startree.aggregator.ValueAggregator; +import org.opensearch.index.codec.startree.aggregator.ValueAggregatorFactory; +import org.opensearch.index.codec.startree.codec.SortedNumericDocValuesWriter; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.node.StarTreeNode; +import org.opensearch.index.codec.startree.util.BufferedAggregatedDocValues; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java index af142ee7cbea8..84c963e8647b7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OffHeapBufferedSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/OffHeapBufferedSingleTreeBuilder.java @@ -14,13 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; @@ -28,8 +27,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RandomAccessInput; import org.apache.lucene.util.IOUtils; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.util.QuickSorter; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.util.QuickSorter; import java.io.IOException; import java.nio.ByteBuffer; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java index 97bfdd5207816..f991f4d95f3ae 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/OnHeapSingleTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/OnHeapSingleTreeBuilder.java @@ -14,15 +14,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java index 43650d98c0871..8ff12372e93e7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/Record.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/Record.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; /** Class representing a star tree record */ public class Record { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java index f86468299d02f..4bd9e136ef322 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/StarTreeBuilderUtils.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/StarTreeBuilderUtils.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,7 +28,7 @@ import java.util.Queue; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.opensearch.index.codec.freshstartree.node.OffHeapStarTreeNode.SERIALIZABLE_SIZE_IN_BYTES; +import static org.opensearch.index.codec.startree.node.OffHeapStarTreeNode.SERIALIZABLE_SIZE_IN_BYTES; /** Util class for building star tree */ public class StarTreeBuilderUtils { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java index 298febc36487b..ccee510fdab6a 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/builder/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/builder/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Builder classes for star tree */ -package org.opensearch.index.codec.freshstartree.builder; +package org.opensearch.index.codec.startree.builder; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java index b02ceb8825597..6d4ab4ba91bce 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesConsumerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesConsumerCopy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java index 3a1eb8eafd533..f9f2ff799b4b8 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/Lucene90DocValuesProducerCopy.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/Lucene90DocValuesProducerCopy.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java index ce2dd81125f46..ece5963b17f82 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/SortedNumericDocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/SortedNumericDocValuesWriter.java @@ -6,18 +6,14 @@ * compatible open source license. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.codecs.DocValuesConsumer; -import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocsWithFieldSet; -import org.apache.lucene.index.EmptyDocValuesProducer; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Sorter; import org.apache.lucene.search.DocIdSetIterator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java similarity index 90% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java index ec971b90ec624..bf6583f8b379e 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeAggregatedValues.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeAggregatedValues.java @@ -14,11 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.opensearch.index.codec.freshstartree.node.StarTree; +import org.opensearch.index.codec.startree.node.StarTree; import java.util.Map; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java index e20deea192733..b73ee8b32c25e 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeCodec.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.CompoundFormat; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java index 7963cb4473d40..6e97cbc54eeba 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesFormat.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesFormat.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java index 8494188c1771e..2d19c80865451 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesReader.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesReader.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; import java.util.LinkedHashMap; import org.apache.lucene.codecs.CodecUtil; @@ -28,8 +28,8 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.store.IndexInput; -import org.opensearch.index.codec.freshstartree.node.OffHeapStarTree; -import org.opensearch.index.codec.freshstartree.node.StarTree; +import org.opensearch.index.codec.startree.node.OffHeapStarTree; +import org.opensearch.index.codec.startree.node.StarTree; import java.io.IOException; import java.util.HashMap; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java similarity index 96% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java index cf393c146fcb3..a4b7510501fd5 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/StarTreeDocValuesWriter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/StarTreeDocValuesWriter.java @@ -14,9 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; -import java.util.Collection; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -27,8 +26,8 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.store.IndexOutput; -import org.opensearch.index.codec.freshstartree.builder.BaseSingleTreeBuilder; -import org.opensearch.index.codec.freshstartree.builder.OffHeapBufferedSingleTreeBuilder; +import org.opensearch.index.codec.startree.builder.BaseSingleTreeBuilder; +import org.opensearch.index.codec.startree.builder.OffHeapBufferedSingleTreeBuilder; import java.io.IOException; import java.util.ArrayList; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java index beea4beb3ddff..05f38a8fb5263 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/codec/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/codec/package-info.java @@ -15,4 +15,4 @@ * limitations under the License. */ /** Codec for performing aggregation during indexing */ -package org.opensearch.index.codec.freshstartree.codec; +package org.opensearch.index.codec.startree.codec; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java index 093c4503debf8..de386eb17c488 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTree.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTree.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java similarity index 99% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java index 3a5b1b55ea9da..87bf130173919 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/OffHeapStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/OffHeapStarTreeNode.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import org.apache.lucene.store.RandomAccessInput; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java index d1892cd564e15..80cdaefb5e3bb 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTree.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTree.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java index cf58d4904cdee..17996b233d1c4 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/StarTreeNode.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; import java.io.IOException; import java.util.Iterator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java similarity index 93% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java index dbd8287650abe..0d81ff3c2ad49 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/node/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/node/package-info.java @@ -16,4 +16,4 @@ */ /** node package */ -package org.opensearch.index.codec.freshstartree.node; +package org.opensearch.index.codec.startree.node; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java index 71b6b4947321d..0aa25aae60fc3 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeFilter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeFilter.java @@ -14,16 +14,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.DocIdSetBuilder; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; -import org.opensearch.index.codec.freshstartree.node.StarTree; -import org.opensearch.index.codec.freshstartree.node.StarTreeNode; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.node.StarTree; +import org.opensearch.index.codec.startree.node.StarTreeNode; import java.io.IOException; import java.util.ArrayDeque; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java index 6bef4545291ef..8a1025f59142c 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQuery.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQuery.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ConstantScoreScorer; @@ -27,7 +27,7 @@ import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Accountable; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import java.io.IOException; import java.util.List; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java index fc2c019557d6f..7f3f519be3227 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/StarTreeQueryBuilder.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/StarTreeQueryBuilder.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -18,7 +18,6 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.codec.freshstartree.builder.BaseSingleTreeBuilder; import org.opensearch.index.query.AbstractQueryBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java index 8b04706698952..f019226645d73 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/query/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/query/package-info.java @@ -16,4 +16,4 @@ */ /** query package - Todo : move it under main dir - keeping it here for readability */ -package org.opensearch.index.codec.freshstartree.query; +package org.opensearch.index.codec.startree.query; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java b/server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java similarity index 97% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java index d3708a1ae8860..92b1d9edf7935 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/BufferedAggregatedDocValues.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/BufferedAggregatedDocValues.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.search.DocIdSetIterator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java b/server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java similarity index 95% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java index 884a4401ade00..9a69963481a7f 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/IntComparator.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/IntComparator.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; import java.util.Comparator; diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java b/server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java similarity index 98% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java index 2e5e33adc06b3..6a9b947204ec7 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/QuickSorter.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/QuickSorter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; /** copy of it.unimi.dsi.fastutil.Arrays.quickSort */ public class QuickSorter { diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java b/server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java index 1df6a7e270698..cc1a4f3f8e721 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/Swapper.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/Swapper.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; /** Functional interface for swapper */ @FunctionalInterface diff --git a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java b/server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java similarity index 94% rename from server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java rename to server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java index 8ad0be6ce1848..4a62aea47bc1b 100644 --- a/server/src/main/java/org/opensearch/index/codec/freshstartree/util/package-info.java +++ b/server/src/main/java/org/opensearch/index/codec/startree/util/package-info.java @@ -16,4 +16,4 @@ */ /** Util package todo: remove this, kept it temporarily for readibility */ -package org.opensearch.index.codec.freshstartree.util; +package org.opensearch.index.codec.startree.util; diff --git a/server/src/main/java/org/opensearch/search/SearchModule.java b/server/src/main/java/org/opensearch/search/SearchModule.java index 29950871b162b..1b26d079164b2 100644 --- a/server/src/main/java/org/opensearch/search/SearchModule.java +++ b/server/src/main/java/org/opensearch/search/SearchModule.java @@ -47,7 +47,7 @@ import org.opensearch.core.common.io.stream.Writeable; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.codec.freshstartree.query.StarTreeQueryBuilder; +import org.opensearch.index.codec.startree.query.StarTreeQueryBuilder; import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.BoostingQueryBuilder; import org.opensearch.index.query.CommonTermsQueryBuilder; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java index f2fd5a45799e2..cfcf2b95cbc47 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/startree/StarTreeAggregator.java @@ -8,13 +8,10 @@ package org.opensearch.search.aggregations.bucket.startree; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Predicate; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -24,7 +21,7 @@ import org.opensearch.core.xcontent.ObjectParser; import org.opensearch.core.xcontent.ToXContentObject; import org.opensearch.core.xcontent.XContentBuilder; -import org.opensearch.index.codec.freshstartree.codec.StarTreeAggregatedValues; +import org.opensearch.index.codec.startree.codec.StarTreeAggregatedValues; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorFactories; import org.opensearch.search.aggregations.CardinalityUpperBound; diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 25df72954122b..3057c566b4658 100644 --- a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -6,4 +6,4 @@ # compatible open source license. # -org.opensearch.index.codec.freshstartree.codec.StarTreeCodec +org.opensearch.index.codec.startree.codec.StarTreeCodec