You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by ma...@apache.org on 2019/06/28 17:41:50 UTC
[incubator-pinot] branch master updated: upgrade fastutil to 8.2.3
(#4325)
This is an automated email from the ASF dual-hosted git repository.
mayanks pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 5e2a810 upgrade fastutil to 8.2.3 (#4325)
5e2a810 is described below
commit 5e2a810bc34fa1f960ce92ac5e7ff1309b854643
Author: Xue Yu <xu...@outlook.com>
AuthorDate: Sat Jun 29 01:41:45 2019 +0800
upgrade fastutil to 8.2.3 (#4325)
---
.../core/data/readers/sort/PinotSegmentSorter.java | 75 +++++++++-------------
.../indexsegment/mutable/MutableSegmentImpl.java | 13 +---
.../aggregation/groupby/utils/DoubleToIdMap.java | 2 +-
.../aggregation/groupby/utils/FloatToIdMap.java | 2 +-
.../aggregation/groupby/utils/IntToIdMap.java | 2 +-
.../aggregation/groupby/utils/LongToIdMap.java | 2 +-
.../pinot/core/startree/StarTreeDataTable.java | 35 ++++------
.../v2/builder/OffHeapSingleTreeBuilder.java | 53 ++++++---------
.../apache/pinot/queries/FastHllQueriesTest.java | 4 +-
.../util/IntDoubleIndexedPriorityQueueTest.java | 2 +-
pom.xml | 2 +-
11 files changed, 67 insertions(+), 125 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/sort/PinotSegmentSorter.java b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/sort/PinotSegmentSorter.java
index d1b53a2..48da081 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/data/readers/sort/PinotSegmentSorter.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/data/readers/sort/PinotSegmentSorter.java
@@ -19,8 +19,6 @@
package org.apache.pinot.core.data.readers.sort;
import it.unimi.dsi.fastutil.Arrays;
-import it.unimi.dsi.fastutil.Swapper;
-import it.unimi.dsi.fastutil.ints.IntComparator;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -84,54 +82,39 @@ public class PinotSegmentSorter implements SegmentSorter {
sortedDocIds[i] = i;
}
- IntComparator comparator = new IntComparator() {
- @Override
- public int compare(int i1, int i2) {
- int docId1 = sortedDocIds[i1];
- int docId2 = sortedDocIds[i2];
-
- int compare = 0;
- for (int index : _sortOrder) {
- String dimensionName = _dimensionNames.get(index);
- FieldSpec fieldSpec = _schema.getFieldSpecFor(dimensionName);
- PinotSegmentColumnReader columnReader = _columnReaderMap.get(dimensionName);
-
- // Multi value column or no dictionary column is not supported
- boolean isMultiValueColumn = !fieldSpec.isSingleValueField();
- boolean isNoDictionaryColumn = !columnReader.hasDictionary();
- if (isMultiValueColumn || isNoDictionaryColumn) {
- throw new IllegalStateException(
- "Multi value column or no dictionary column is not supported. ( column name: " + dimensionName
- + ", multi value column: " + isMultiValueColumn + ", no dictionary column: " + isNoDictionaryColumn
- + " )");
- }
-
- // Compute the order
- compare = columnReader.getDictionaryId(docId1) - columnReader.getDictionaryId(docId2);
-
- if (compare != 0) {
- return compare;
- }
+ Arrays.quickSort(0, _numDocs, (i1, i2) -> {
+ int docId1 = sortedDocIds[i1];
+ int docId2 = sortedDocIds[i2];
+
+ int compare = 0;
+ for (int sortIndex : _sortOrder) {
+ String dimensionName = _dimensionNames.get(sortIndex);
+ FieldSpec fieldSpec = _schema.getFieldSpecFor(dimensionName);
+ PinotSegmentColumnReader columnReader = _columnReaderMap.get(dimensionName);
+
+ // Multi value column or no dictionary column is not supported
+ boolean isMultiValueColumn = !fieldSpec.isSingleValueField();
+ boolean isNoDictionaryColumn = !columnReader.hasDictionary();
+ if (isMultiValueColumn || isNoDictionaryColumn) {
+ throw new IllegalStateException(
+ "Multi value column or no dictionary column is not supported. ( column name: " + dimensionName
+ + ", multi value column: " + isMultiValueColumn + ", no dictionary column: " + isNoDictionaryColumn
+ + " )");
}
- return compare;
- }
- @Override
- public int compare(Integer o1, Integer o2) {
- throw new UnsupportedOperationException();
- }
- };
+ // Compute the order
+ compare = columnReader.getDictionaryId(docId1) - columnReader.getDictionaryId(docId2);
- Swapper swapper = new Swapper() {
- @Override
- public void swap(int i, int j) {
- int temp = sortedDocIds[i];
- sortedDocIds[i] = sortedDocIds[j];
- sortedDocIds[j] = temp;
+ if (compare != 0) {
+ return compare;
+ }
}
- };
-
- Arrays.quickSort(0, _numDocs, comparator, swapper);
+ return compare;
+ }, (i, j) -> {
+ int temp = sortedDocIds[i];
+ sortedDocIds[i] = sortedDocIds[j];
+ sortedDocIds[j] = temp;
+ });
return sortedDocIds;
}
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
index 972d91c..457d8c7 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/indexsegment/mutable/MutableSegmentImpl.java
@@ -20,7 +20,6 @@ package org.apache.pinot.core.indexsegment.mutable;
import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.ints.IntArrays;
-import it.unimi.dsi.fastutil.ints.IntComparator;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -532,18 +531,8 @@ public class MutableSegmentImpl implements MutableSegment {
for (int i = 0; i < numValues; i++) {
dictIds[i] = i;
}
- IntArrays.quickSort(dictIds, new IntComparator() {
- @Override
- public int compare(int dictId1, int dictId2) {
- return dictionary.compare(dictId1, dictId2);
- }
-
- @Override
- public int compare(Integer o1, Integer o2) {
- return compare((int) o1, (int) o2);
- }
- });
+ IntArrays.quickSort(dictIds, (dictId1, dictId2) -> dictionary.compare(dictId1, dictId2));
RealtimeInvertedIndexReader invertedIndex = _invertedIndexMap.get(column);
int[] docIds = new int[_numDocsIndexed];
int docIdIndex = 0;
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/DoubleToIdMap.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/DoubleToIdMap.java
index 070ca2c..c882c3d 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/DoubleToIdMap.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/DoubleToIdMap.java
@@ -51,7 +51,7 @@ public class DoubleToIdMap extends BaseValueToIdMap {
@Override
public double getDouble(int id) {
assert id < _idToValueMap.size();
- return _idToValueMap.get(id);
+ return _idToValueMap.getDouble(id);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/FloatToIdMap.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/FloatToIdMap.java
index 77b625b..0f21c2c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/FloatToIdMap.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/FloatToIdMap.java
@@ -51,7 +51,7 @@ public class FloatToIdMap extends BaseValueToIdMap {
@Override
public float getFloat(int id) {
assert id < _idToValueMap.size();
- return _idToValueMap.get(id);
+ return _idToValueMap.getFloat(id);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/IntToIdMap.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/IntToIdMap.java
index cdb1795..71b4025 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/IntToIdMap.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/IntToIdMap.java
@@ -51,7 +51,7 @@ public class IntToIdMap extends BaseValueToIdMap {
@Override
public int getInt(int id) {
assert id < _idToValueMap.size();
- return _idToValueMap.get(id);
+ return _idToValueMap.getInt(id);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/LongToIdMap.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/LongToIdMap.java
index 859a3c2..a498662 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/LongToIdMap.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/utils/LongToIdMap.java
@@ -51,7 +51,7 @@ public class LongToIdMap extends BaseValueToIdMap {
@Override
public long getLong(int id) {
assert id < _idToValueMap.size();
- return _idToValueMap.get(id);
+ return _idToValueMap.getLong(id);
}
@Override
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeDataTable.java b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeDataTable.java
index d3e880a..b70505c 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeDataTable.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/startree/StarTreeDataTable.java
@@ -20,10 +20,8 @@ package org.apache.pinot.core.startree;
import com.google.common.base.Preconditions;
import it.unimi.dsi.fastutil.Arrays;
-import it.unimi.dsi.fastutil.Swapper;
import it.unimi.dsi.fastutil.ints.Int2ObjectLinkedOpenHashMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
-import it.unimi.dsi.fastutil.ints.IntComparator;
import java.io.Closeable;
import java.io.IOException;
import java.util.Iterator;
@@ -84,33 +82,22 @@ public class StarTreeDataTable implements Closeable {
sortedDocIds[i] = i + startDocIdOffset;
}
- IntComparator comparator = new IntComparator() {
- @Override
- public int compare(int i1, int i2) {
- long offset1 = sortedDocIds[i1] * _docSizeLong;
- long offset2 = sortedDocIds[i2] * _docSizeLong;
- for (int index : sortOrder) {
- int v1 = _dataBuffer.getInt(offset1 + index * Integer.BYTES);
- int v2 = _dataBuffer.getInt(offset2 + index * Integer.BYTES);
- if (v1 != v2) {
- return v1 - v2;
- }
+ Arrays.quickSort(0, numDocs, (i1, i2) -> {
+ long offset1 = sortedDocIds[i1] * _docSizeLong;
+ long offset2 = sortedDocIds[i2] * _docSizeLong;
+ for (int index : sortOrder) {
+ int v1 = _dataBuffer.getInt(offset1 + index * Integer.BYTES);
+ int v2 = _dataBuffer.getInt(offset2 + index * Integer.BYTES);
+ if (v1 != v2) {
+ return v1 - v2;
}
- return 0;
- }
-
- @Override
- public int compare(Integer o1, Integer o2) {
- throw new UnsupportedOperationException();
}
- };
-
- Swapper swapper = (i, j) -> {
+ return 0;
+ }, (i, j) -> {
int temp = sortedDocIds[i];
sortedDocIds[i] = sortedDocIds[j];
sortedDocIds[j] = temp;
- };
- Arrays.quickSort(0, numDocs, comparator, swapper);
+ });
// Re-arrange documents based on the sorted document ids
// Each write places a document in it's proper location, so time complexity is O(n)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/startree/v2/builder/OffHeapSingleTreeBuilder.java b/pinot-core/src/main/java/org/apache/pinot/core/startree/v2/builder/OffHeapSingleTreeBuilder.java
index 0e658c6..a0de341 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/startree/v2/builder/OffHeapSingleTreeBuilder.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/startree/v2/builder/OffHeapSingleTreeBuilder.java
@@ -19,7 +19,6 @@
package org.apache.pinot.core.startree.v2.builder;
import com.google.common.base.Preconditions;
-import it.unimi.dsi.fastutil.ints.IntComparator;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileNotFoundException;
@@ -217,25 +216,17 @@ public class OffHeapSingleTreeBuilder extends BaseSingleTreeBuilder {
offset += Integer.BYTES;
}
}
- it.unimi.dsi.fastutil.Arrays.quickSort(0, numDocs, new IntComparator() {
- @Override
- public int compare(int i1, int i2) {
- long offset1 = (long) sortedDocIds[i1] * _numDimensions * Integer.BYTES;
- long offset2 = (long) sortedDocIds[i2] * _numDimensions * Integer.BYTES;
- for (int i = 0; i < _numDimensions; i++) {
- int dimension1 = dataBuffer.getInt(offset1 + i * Integer.BYTES);
- int dimension2 = dataBuffer.getInt(offset2 + i * Integer.BYTES);
- if (dimension1 != dimension2) {
- return dimension1 - dimension2;
- }
+ it.unimi.dsi.fastutil.Arrays.quickSort(0, numDocs, (i1, i2) -> {
+ long offset1 = (long) sortedDocIds[i1] * _numDimensions * Integer.BYTES;
+ long offset2 = (long) sortedDocIds[i2] * _numDimensions * Integer.BYTES;
+ for (int i = 0; i < _numDimensions; i++) {
+ int dimension1 = dataBuffer.getInt(offset1 + i * Integer.BYTES);
+ int dimension2 = dataBuffer.getInt(offset2 + i * Integer.BYTES);
+ if (dimension1 != dimension2) {
+ return dimension1 - dimension2;
}
- return 0;
- }
-
- @Override
- public int compare(Integer o1, Integer o2) {
- throw new UnsupportedOperationException();
}
+ return 0;
}, (i1, i2) -> {
int temp = sortedDocIds[i1];
sortedDocIds[i1] = sortedDocIds[i2];
@@ -288,25 +279,17 @@ public class OffHeapSingleTreeBuilder extends BaseSingleTreeBuilder {
for (int i = 0; i < numDocs; i++) {
sortedDocIds[i] = startDocId + i;
}
- it.unimi.dsi.fastutil.Arrays.quickSort(0, numDocs, new IntComparator() {
- @Override
- public int compare(int i1, int i2) {
- long offset1 = _starTreeRecordOffsets.get(sortedDocIds[i1]);
- long offset2 = _starTreeRecordOffsets.get(sortedDocIds[i2]);
- for (int i = dimensionId + 1; i < _numDimensions; i++) {
- int dimension1 = _starTreeRecordBuffer.getInt(offset1 + i * Integer.BYTES);
- int dimension2 = _starTreeRecordBuffer.getInt(offset2 + i * Integer.BYTES);
- if (dimension1 != dimension2) {
- return dimension1 - dimension2;
- }
+ it.unimi.dsi.fastutil.Arrays.quickSort(0, numDocs, (i1, i2) -> {
+ long offset1 = _starTreeRecordOffsets.get(sortedDocIds[i1]);
+ long offset2 = _starTreeRecordOffsets.get(sortedDocIds[i2]);
+ for (int i = dimensionId + 1; i < _numDimensions; i++) {
+ int dimension1 = _starTreeRecordBuffer.getInt(offset1 + i * Integer.BYTES);
+ int dimension2 = _starTreeRecordBuffer.getInt(offset2 + i * Integer.BYTES);
+ if (dimension1 != dimension2) {
+ return dimension1 - dimension2;
}
- return 0;
- }
-
- @Override
- public int compare(Integer o1, Integer o2) {
- throw new UnsupportedOperationException();
}
+ return 0;
}, (i1, i2) -> {
int temp = sortedDocIds[i1];
sortedDocIds[i1] = sortedDocIds[i2];
diff --git a/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java b/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
index 57f0171..dbf0b9b 100644
--- a/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/queries/FastHllQueriesTest.java
@@ -121,7 +121,7 @@ public class FastHllQueriesTest extends BaseQueriesTest {
aggregationOperator = getOperatorForQueryWithFilter(BASE_QUERY);
resultsBlock = aggregationOperator.nextBlock();
executionStatistics = aggregationOperator.getExecutionStatistics();
- QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 6129L, 112472L, 12258L, 30000L);
+ QueriesTestUtils.testInnerSegmentExecutionStatistics(executionStatistics, 6129L, 112479L, 12258L, 30000L);
aggregationResult = resultsBlock.getAggregationResult();
Assert.assertEquals(((HyperLogLog) aggregationResult.get(0)).cardinality(), 17L);
Assert.assertEquals(((HyperLogLog) aggregationResult.get(1)).cardinality(), 1197L);
@@ -141,7 +141,7 @@ public class FastHllQueriesTest extends BaseQueriesTest {
QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 4L, 0L, 8L, 120000L, new String[]{"21", "1762"});
// Test inter segments query with filter
brokerResponse = getBrokerResponseForQueryWithFilter(BASE_QUERY);
- QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 24516L, 449888L, 49032L, 120000L,
+ QueriesTestUtils.testInterSegmentAggregationResult(brokerResponse, 24516L, 449916L, 49032L, 120000L,
new String[]{"17", "1197"});
// Test inter segments query with group-by
brokerResponse = getBrokerResponseForQuery(BASE_QUERY + GROUP_BY);
diff --git a/pinot-core/src/test/java/org/apache/pinot/util/IntDoubleIndexedPriorityQueueTest.java b/pinot-core/src/test/java/org/apache/pinot/util/IntDoubleIndexedPriorityQueueTest.java
index bab2ea0..c7ab217 100644
--- a/pinot-core/src/test/java/org/apache/pinot/util/IntDoubleIndexedPriorityQueueTest.java
+++ b/pinot-core/src/test/java/org/apache/pinot/util/IntDoubleIndexedPriorityQueueTest.java
@@ -85,7 +85,7 @@ public class IntDoubleIndexedPriorityQueueTest {
// Transfer the map into list so it can be sorted.
List<Pairs.IntDoublePair> list = new ArrayList<>(NUM_RECORDS);
for (Int2DoubleMap.Entry entry : map.int2DoubleEntrySet()) {
- list.add(new Pairs.IntDoublePair(entry.getKey(), entry.getValue()));
+ list.add(new Pairs.IntDoublePair(entry.getIntKey(), entry.getDoubleValue()));
}
// Comparison for min heap is the same as that for ascending order.
diff --git a/pom.xml b/pom.xml
index 4e27925..7e8ba3b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -367,7 +367,7 @@
<dependency>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
- <version>6.5.16</version>
+ <version>8.2.3</version>
</dependency>
<dependency>
<groupId>joda-time</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org