You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by xi...@apache.org on 2020/05/20 06:41:57 UTC
[incubator-pinot] branch thread_local_hashmap created (now 0b2fd80)
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a change to branch thread_local_hashmap
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git.
at 0b2fd80 Use thread local for groupby raw key holders
This branch includes the following new commits:
new 0b2fd80 Use thread local for groupby raw key holders
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org
[incubator-pinot] 01/01: Use thread local for groupby raw key
holders
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch thread_local_hashmap
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0b2fd80c7ee13268b229fe4269d2e2df827c3675
Author: Xiang Fu <fx...@gmail.com>
AuthorDate: Tue May 19 23:41:39 2020 -0700
Use thread local for groupby raw key holders
---
.../groupby/DictionaryBasedGroupKeyGenerator.java | 26 ++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGenerator.java b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGenerator.java
index e2da862..29d6c26 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGenerator.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/query/aggregation/groupby/DictionaryBasedGroupKeyGenerator.java
@@ -25,7 +25,9 @@ import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap;
import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
import it.unimi.dsi.fastutil.objects.ObjectIterator;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import java.util.NoSuchElementException;
import javax.annotation.Nonnull;
import org.apache.pinot.common.request.transform.TransformExpressionTree;
@@ -75,6 +77,7 @@ public class DictionaryBasedGroupKeyGenerator implements GroupKeyGenerator {
private final int _globalGroupIdUpperBound;
private final RawKeyHolder _rawKeyHolder;
+ private final ThreadLocal<Map<String, RawKeyHolder>> _mapBasedRawKeyHolders = new ThreadLocal<>();
public DictionaryBasedGroupKeyGenerator(TransformOperator transformOperator,
TransformExpressionTree[] groupByExpressions, int numGroupsLimit, int arrayBasedThreshold) {
@@ -107,19 +110,34 @@ public class DictionaryBasedGroupKeyGenerator implements GroupKeyGenerator {
_isSingleValueColumn[i] = transformOperator.getResultMetadata(groupByExpression).isSingleValue();
}
+ if (_mapBasedRawKeyHolders.get() == null) {
+ _mapBasedRawKeyHolders.set(new HashMap<>());
+ }
if (longOverflow) {
_globalGroupIdUpperBound = numGroupsLimit;
- _rawKeyHolder = new ArrayMapBasedHolder(_globalGroupIdUpperBound);
+ if (!_mapBasedRawKeyHolders.get().containsKey("longOverflow")) {
+ _mapBasedRawKeyHolders.get().put("longOverflow", new ArrayMapBasedHolder(_globalGroupIdUpperBound));
+ }
+ _rawKeyHolder = _mapBasedRawKeyHolders.get().get("longOverflow");
} else {
if (cardinalityProduct > Integer.MAX_VALUE) {
_globalGroupIdUpperBound = numGroupsLimit;
- _rawKeyHolder = new LongMapBasedHolder(_globalGroupIdUpperBound);
+ if (!_mapBasedRawKeyHolders.get().containsKey("long")) {
+ _mapBasedRawKeyHolders.get().put("long", new LongMapBasedHolder(_globalGroupIdUpperBound));
+ }
+ _rawKeyHolder = _mapBasedRawKeyHolders.get().get("long");
} else {
_globalGroupIdUpperBound = Math.min((int) cardinalityProduct, numGroupsLimit);
if (cardinalityProduct > arrayBasedThreshold) {
- _rawKeyHolder = new IntMapBasedHolder(_globalGroupIdUpperBound);
+ if (!_mapBasedRawKeyHolders.get().containsKey("int")) {
+ _mapBasedRawKeyHolders.get().put("int", new IntMapBasedHolder(_globalGroupIdUpperBound));
+ }
+ _rawKeyHolder = _mapBasedRawKeyHolders.get().get("int");
} else {
- _rawKeyHolder = new ArrayBasedHolder();
+ if (!_mapBasedRawKeyHolders.get().containsKey("array")) {
+ _mapBasedRawKeyHolders.get().put("array", new ArrayBasedHolder());
+ }
+ _rawKeyHolder = _mapBasedRawKeyHolders.get().get("array");
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org