You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/03/06 03:43:03 UTC
git commit: Make default estimation for number of distinct group by
values configurable
Repository: incubator-phoenix
Updated Branches:
refs/heads/master 9298be876 -> c22ea0996
Make default estimation for number of distinct group by values configurable
Project: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/commit/c22ea099
Tree: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/tree/c22ea099
Diff: http://git-wip-us.apache.org/repos/asf/incubator-phoenix/diff/c22ea099
Branch: refs/heads/master
Commit: c22ea0996dd59f0ad794836ab4e7cca71f8c5ccd
Parents: 9298be8
Author: James Taylor <ja...@apache.org>
Authored: Wed Mar 5 18:41:07 2014 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Wed Mar 5 18:41:07 2014 -0800
----------------------------------------------------------------------
.../GroupedAggregateRegionObserver.java | 19 ++++++++++---------
.../org/apache/phoenix/query/QueryServices.java | 1 +
.../phoenix/query/QueryServicesOptions.java | 1 +
3 files changed, 12 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c22ea099/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
index d6c0063..583d6c5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupedAggregateRegionObserver.java
@@ -20,7 +20,9 @@ package org.apache.phoenix.coprocessor;
import static org.apache.phoenix.query.QueryConstants.AGG_TIMESTAMP;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryServices.GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILLABLE_ATTRIB;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_GROUPBY_SPILLABLE;
import java.io.ByteArrayInputStream;
@@ -47,11 +49,6 @@ import org.apache.hadoop.hbase.regionserver.MultiVersionConsistencyControl;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.WritableUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.collect.Maps;
-import com.google.common.io.Closeables;
import org.apache.phoenix.cache.GlobalCache;
import org.apache.phoenix.cache.TenantCache;
import org.apache.phoenix.cache.aggcache.SpillableGroupByCache;
@@ -69,6 +66,11 @@ import org.apache.phoenix.util.KeyValueUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SizedUtil;
import org.apache.phoenix.util.TupleUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.collect.Maps;
+import com.google.common.io.Closeables;
/**
* Region observer that aggregates grouped rows (i.e. SQL query with GROUP BY clause)
@@ -84,7 +86,6 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
public static final String KEY_ORDERED_GROUP_BY_EXPRESSIONS = "OrderedGroupByExpressions";
public static final String ESTIMATED_DISTINCT_VALUES = "EstDistinctValues";
- public static final int DEFAULT_ESTIMATED_DISTINCT_VALUES = 10000;
public static final int MIN_DISTINCT_VALUES = 100;
/**
@@ -338,7 +339,9 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
logger.debug("Grouped aggregation over unordered rows with scan " + scan
+ ", group by " + expressions + ", aggregators " + aggregators);
}
- int estDistVals = DEFAULT_ESTIMATED_DISTINCT_VALUES;
+ RegionCoprocessorEnvironment env = c.getEnvironment();
+ Configuration conf = env.getConfiguration();
+ int estDistVals = conf.getInt(GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB, DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES);
byte[] estDistValsBytes = scan.getAttribute(ESTIMATED_DISTINCT_VALUES);
if (estDistValsBytes != null) {
// Allocate 1.5x estimation
@@ -346,8 +349,6 @@ public class GroupedAggregateRegionObserver extends BaseScannerRegionObserver {
(int) (Bytes.toInt(estDistValsBytes) * 1.5f));
}
- RegionCoprocessorEnvironment env = c.getEnvironment();
- Configuration conf = env.getConfiguration();
final boolean spillableEnabled =
conf.getBoolean(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE);
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c22ea099/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 0210c58..75d64cf 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -80,6 +80,7 @@ public interface QueryServices extends SQLCloseable {
public static final String GROUPBY_SPILLABLE_ATTRIB = "phoenix.groupby.spillable";
public static final String GROUPBY_SPILL_FILES_ATTRIB = "phoenix.groupby.spillFiles";
public static final String GROUPBY_MAX_CACHE_SIZE_ATTRIB = "phoenix.groupby.maxCacheSize";
+ public static final String GROUPBY_ESTIMATED_DISTINCT_VALUES_ATTRIB = "phoenix.groupby.estimatedDistinctValues";
public static final String CALL_QUEUE_PRODUCER_ATTRIB_NAME = "CALL_QUEUE_PRODUCER";
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/c22ea099/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 4af0871..fbed133 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -116,6 +116,7 @@ public class QueryServicesOptions {
public static final long DEFAULT_MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS = 60000 * 30; // 30 mins
public static final long DEFAULT_MAX_SERVER_METADATA_CACHE_SIZE = 1024L*1024L*20L; // 20 Mb
public static final long DEFAULT_MAX_CLIENT_METADATA_CACHE_SIZE = 1024L*1024L*10L; // 10 Mb
+ public static final int DEFAULT_GROUPBY_ESTIMATED_DISTINCT_VALUES = 1000;
private final Configuration config;