You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2016/05/27 17:46:28 UTC
[1/4] incubator-trafodion git commit: [TRAFODION-2017] Use a maximum
HBase row cache of 50 for USTAT sample scans
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 8a59ad263 -> 1fa468760
[TRAFODION-2017] Use a maximum HBase row cache of 50 for USTAT sample scans
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d03526a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d03526a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d03526a2
Branch: refs/heads/master
Commit: d03526a2a508097dc8f23ac6fd63abb756a2a98e
Parents: 45c2e76
Author: Dave Birdsall <db...@apache.org>
Authored: Thu May 26 22:03:10 2016 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu May 26 22:03:10 2016 +0000
----------------------------------------------------------------------
core/sql/ustat/hs_globals.cpp | 22 +++++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d03526a2/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index 06b86ad..c32e6cc 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -529,8 +529,19 @@ Int32 getMissingCols (const NABitVector &map1, const NABitVector &map2, NABitVec
// done without a timeout, trading off possible speed improvements
// by using a smaller cache size.
//
-// Note that when we move to HBase 1.1, with its heartbeat protocol,
-// this time-out problem goes away and we can remove these CQDs.
+// Another issue is that it's been observed that time-outs also
+// depend on system load. Through experimentation we've discovered
+// that a value of 50 works well in loaded scenarios, assuming the
+// table is not too large. So, we use a maximum of the workable
+// value computed above and 50.
+//
+// Another note: If the user has already set HBASE_NUM_CACHE_ROWS_MAX,
+// then we don't do anything here. We respect the user's choices
+// instead.
+//
+// We had hoped that HBase 1.1, with its heartbeat protocol, would
+// solve this time-out problem for good. But early testing seems
+// to suggest that this is not the case.
//
// Input:
// sampleRatio -- Percentage of rows being sampled.
@@ -546,10 +557,11 @@ NABoolean HSGlobalsClass::setHBaseCacheSize(double sampleRatio)
Int64 workableCacheSize = (Int64)(sampleRatio * calibrationFactor);
if (workableCacheSize < 1)
workableCacheSize = 1; // can't go below 1 unfortunately
+ else if (workableCacheSize > 50)
+ workableCacheSize = 50;
- Int32 max = getDefaultAsLong(HBASE_NUM_CACHE_ROWS_MAX);
- if ((workableCacheSize < 10000) && // don't bother if 10000 works
- (max == 10000)) // don't do it if user has already set this CQD
+ Int32 maxDefault = getDefaultAsLong(HBASE_NUM_CACHE_ROWS_MAX);
+ if (maxDefault == 10000) // don't do it if user has already set this CQD
{
char temp1[40]; // way more space than needed, but it's safe
Lng32 wcs = (Lng32)workableCacheSize;
[4/4] incubator-trafodion git commit: Merge [TRAFODION-2017] PR 505
Limit HBase row cache to 50 for USTAT sampling
Posted by db...@apache.org.
Merge [TRAFODION-2017] PR 505 Limit HBase row cache to 50 for USTAT sampling
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1fa46876
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1fa46876
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1fa46876
Branch: refs/heads/master
Commit: 1fa46876069ee0844c27677b4c2a46b2327d9ec4
Parents: 8a59ad2 16aa143
Author: Dave Birdsall <db...@apache.org>
Authored: Fri May 27 17:45:32 2016 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Fri May 27 17:45:32 2016 +0000
----------------------------------------------------------------------
core/sql/ustat/hs_globals.cpp | 28 ++++++++++++++++++++++------
1 file changed, 22 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[2/4] incubator-trafodion git commit: Rework
Posted by db...@apache.org.
Rework
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/cacb31ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/cacb31ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/cacb31ef
Branch: refs/heads/master
Commit: cacb31ef5cfa8711259a4aa00de55908ba51e3cf
Parents: d03526a
Author: Dave Birdsall <db...@apache.org>
Authored: Thu May 26 23:44:52 2016 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Thu May 26 23:44:52 2016 +0000
----------------------------------------------------------------------
core/sql/ustat/hs_globals.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/cacb31ef/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index c32e6cc..67c29f5 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -560,8 +560,10 @@ NABoolean HSGlobalsClass::setHBaseCacheSize(double sampleRatio)
else if (workableCacheSize > 50)
workableCacheSize = 50;
- Int32 maxDefault = getDefaultAsLong(HBASE_NUM_CACHE_ROWS_MAX);
- if (maxDefault == 10000) // don't do it if user has already set this CQD
+ // if the user himself set the CQD, don't do anything
+ NADefaults &defs = ActiveSchemaDB()->getDefaults();
+ if (defs.getProvenance(HBASE_NUM_CACHE_ROWS_MAX) ==
+ NADefaults::INIT_DEFAULT_DEFAULTS)
{
char temp1[40]; // way more space than needed, but it's safe
Lng32 wcs = (Lng32)workableCacheSize;
[3/4] incubator-trafodion git commit: Rework number 2
Posted by db...@apache.org.
Rework number 2
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/16aa1431
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/16aa1431
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/16aa1431
Branch: refs/heads/master
Commit: 16aa1431210ec8de3015f1f6905dc54f4802b532
Parents: cacb31e
Author: Dave Birdsall <db...@apache.org>
Authored: Fri May 27 15:34:36 2016 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Fri May 27 15:34:36 2016 +0000
----------------------------------------------------------------------
core/sql/ustat/hs_globals.cpp | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/16aa1431/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index 67c29f5..50c15d5 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -560,10 +560,12 @@ NABoolean HSGlobalsClass::setHBaseCacheSize(double sampleRatio)
else if (workableCacheSize > 50)
workableCacheSize = 50;
- // if the user himself set the CQD, don't do anything
+ // Do this only if the user didn't set the CQD in this session
+ // (So, for example, if the CQD was set in the DEFAULTS table
+ // but not in this session, we'll still override it.)
NADefaults &defs = ActiveSchemaDB()->getDefaults();
- if (defs.getProvenance(HBASE_NUM_CACHE_ROWS_MAX) ==
- NADefaults::INIT_DEFAULT_DEFAULTS)
+ if (defs.getProvenance(HBASE_NUM_CACHE_ROWS_MAX) <
+ NADefaults::SET_BY_CQD)
{
char temp1[40]; // way more space than needed, but it's safe
Lng32 wcs = (Lng32)workableCacheSize;