You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2017/12/12 19:19:11 UTC
[1/3] incubator-trafodion git commit: [TRAFODION-2420] RMS
enhancements
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 160785a30 -> 749537783
[TRAFODION-2420] RMS enhancements
The float type counters such as Estimated Accesssed Rows, Estimated Rows used and
Estimated Memory are not formatted correctly after sprintf changes.
This resulted in RMS output to be messed up making it diffcult to read and analyze
the output.
(cherry picked from commit 41c0ca7067c1e0097c05e1c823ea26eb2a828591)
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1a3dc674
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1a3dc674
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1a3dc674
Branch: refs/heads/master
Commit: 1a3dc6740be73800cfe899632deaa0a8e4f4d64f
Parents: 19c7544
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Dec 8 23:00:44 2017 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Fri Dec 8 23:00:44 2017 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 2 -
core/sql/executor/ExExeUtilGetStats.cpp | 64 ++++++----------------------
2 files changed, 13 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a3dc674/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 9cad737..2965d21 100644
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -2280,7 +2280,6 @@ private:
SQLSTATS_ITEM *masterStatsItems_;
SQLSTATS_ITEM *measStatsItems_;
SQLSTATS_ITEM* operatorStatsItems_;
- SQLSTATS_ITEM* dp2OperatorStatsItems_;
SQLSTATS_ITEM *rootOperStatsItems_;
SQLSTATS_ITEM* partitionAccessStatsItems_;
SQLSTATS_ITEM *pertableStatsItems_;
@@ -2295,7 +2294,6 @@ private:
Lng32 maxMasterStatsItems_;
Lng32 maxMeasStatsItems_;
Lng32 maxOperatorStatsItems_;
- Lng32 maxDp2OperatorStatsItems_;
Lng32 maxRootOperStatsItems_;
Lng32 maxPartitionAccessStatsItems_;
Lng32 maxPertableStatsItems_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1a3dc674/core/sql/executor/ExExeUtilGetStats.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilGetStats.cpp b/core/sql/executor/ExExeUtilGetStats.cpp
index 8f9a3cd..469f6fd 100644
--- a/core/sql/executor/ExExeUtilGetStats.cpp
+++ b/core/sql/executor/ExExeUtilGetStats.cpp
@@ -219,8 +219,6 @@ static const QueryString getStatsAllDefaultViewQuery[] =
{" when tdb_name = 'EX_SEND_TOP' or tdb_name = 'EX_SEND_BOTTOM' "},
{" or tdb_name = 'EX_HASHJ' or tdb_name = 'EX_HASH_GRBY' "},
{" or tdb_name = 'EX_SORT' "},
-{" or tdb_name = 'EX_DP2_SUBS_OPER' "},
-{" or tdb_name = 'EX_DP2_UNIQUE_OPER' "},
{" or tdb_name = 'EX_SPLIT_TOP' "},
{" or tdb_name = 'EX_FAST_EXTRACT' "},
{" then "},
@@ -585,21 +583,12 @@ short ExExeUtilGetStatisticsTcb::work()
FALSE, TRUE);
strcpy(statsBuf_, "Rows ReturnedToUser: ");
strcat(statsBuf_, formattedFloatVal);
- strcat(statsBuf_, " AccessedInDp2: ");
- FormatFloat(formattedFloatVal, intSize, valSize,
- csi.dp2RowsAccessed(), FALSE, TRUE);
- strcat(statsBuf_, formattedFloatVal);
- strcat(statsBuf_, " UsedInDp2: ");
- FormatFloat(formattedFloatVal, intSize, valSize,
- csi.dp2RowsUsed(), FALSE, TRUE);
- strcat(statsBuf_, formattedFloatVal);
moveRowToUpQueue(statsBuf_);
- sprintf(statsBuf_, "Fragment Size(Kb) Total: %-4d Master: %-4d ESP: %-4d DP2: %-4d",
+ sprintf(statsBuf_, "Fragment Size(Kb) Total: %-4d Master: %-4d ESP: %-4d ",
csi.totalFragmentSize(),
csi.masterFragmentSize(),
- csi.espFragmentSize(),
- csi.dp2FragmentSize());
+ csi.espFragmentSize());
moveRowToUpQueue(statsBuf_);
sprintf(statsBuf_, "Operators Total: %-3d",
@@ -609,8 +598,8 @@ short ExExeUtilGetStatisticsTcb::work()
sprintf(statsBuf_, " Joins HJ: %-3d MJ: %-3d NJ: %-3d Total: %-3d",
csi.hj(), csi.mj(), csi.nj(), csi.totalJoins());
moveRowToUpQueue(statsBuf_);
- sprintf(statsBuf_, " Others ESPExchange: %-3d DP2Oper: %-3d UDR: %-3d BMO: %-3d",
- csi.exchangeOps(), csi.dp2Ops(), csi.udr(), csi.bmo());
+ sprintf(statsBuf_, " Others ESPExchange: %-3d UDR: %-3d BMO: %-3d",
+ csi.exchangeOps(), csi.udr(), csi.bmo());
moveRowToUpQueue(statsBuf_);
@@ -1332,22 +1321,6 @@ short ExExeUtilGetStatisticsTcb::work()
sprintf(statsBuf_, "%-25s%10sKB", "SQL Heap Used", sstrbuf);
moveRowToUpQueue(statsBuf_);
- getSubstrInfo(statsRow_, statsRowlen_, "Dp2SpaceTotal:", sstrbuf);
- sprintf(statsBuf_, "%-25s%10sKB", "EID Space Allocated", sstrbuf);
- moveRowToUpQueue(statsBuf_);
-
- getSubstrInfo(statsRow_, statsRowlen_, "Dp2SpaceUsed:", sstrbuf);
- sprintf(statsBuf_, "%-25s%10sKB", "EID Space Used", sstrbuf);
- moveRowToUpQueue(statsBuf_);
-
- getSubstrInfo(statsRow_, statsRowlen_, "Dp2HeapTotal:", sstrbuf);
- sprintf(statsBuf_, "%-25s%10sKB", "EID Heap Allocated", sstrbuf);
- moveRowToUpQueue(statsBuf_);
-
- getSubstrInfo(statsRow_, statsRowlen_, "Dp2HeapUsed:", sstrbuf);
- sprintf(statsBuf_, "%-25s%10sKB", "EID Heap Used", sstrbuf);
- moveRowToUpQueue(statsBuf_);
-
getSubstrInfo(statsRow_, statsRowlen_, "Opens:", sstrbuf);
sprintf(statsBuf_, "%-25s%10s", "Opens", sstrbuf);
moveRowToUpQueue(statsBuf_);
@@ -1471,7 +1444,6 @@ ExExeUtilGetRTSStatisticsTcb::ExExeUtilGetRTSStatisticsTcb(
masterStatsItems_ = NULL;
measStatsItems_ = NULL;
operatorStatsItems_ = NULL;
- dp2OperatorStatsItems_ = NULL;
rootOperStatsItems_ = NULL;
partitionAccessStatsItems_ = NULL;
pertableStatsItems_ = NULL;
@@ -1490,7 +1462,6 @@ ExExeUtilGetRTSStatisticsTcb::ExExeUtilGetRTSStatisticsTcb(
maxMasterStatsItems_ = 0;
maxMeasStatsItems_ = 0;
maxOperatorStatsItems_ = 0;
- maxDp2OperatorStatsItems_ = 0;
maxRootOperStatsItems_ = 0;
maxPartitionAccessStatsItems_ = 0;
maxPertableStatsItems_ = 0;
@@ -1519,11 +1490,6 @@ ExExeUtilGetRTSStatisticsTcb::~ExExeUtilGetRTSStatisticsTcb()
deleteSqlStatItems(measStatsItems_, maxMeasStatsItems_);
measStatsItems_ = NULL;
}
- if (dp2OperatorStatsItems_ != NULL)
- {
- deleteSqlStatItems(dp2OperatorStatsItems_, maxDp2OperatorStatsItems_);
- dp2OperatorStatsItems_ = NULL;
- }
if (operatorStatsItems_ != NULL)
{
deleteSqlStatItems(operatorStatsItems_, maxOperatorStatsItems_);
@@ -1705,8 +1671,7 @@ void ExExeUtilGetRTSStatisticsTcb::formatOperStats(SQLSTATS_ITEM* operStatsItems
sprintf(&statsBuf_[strlen(statsBuf_)], "%19s", valString);
// Estimated Rows Used
- FormatFloat(valString, intSize, valSize, operStatsItems[9].double_value,
- FALSE, TRUE);
+ sprintf(valString, "%.6g", operStatsItems[9].double_value);
sprintf(&statsBuf_[strlen(statsBuf_)], "%19s", valString);
// Actual Rows Used
@@ -2232,15 +2197,10 @@ short ExExeUtilGetRTSStatisticsTcb::work()
ComTdbRoot::getSubqueryTypeText(subqueryType));
break;
case SQLSTATS_EST_ROWS_ACCESSED:
- FormatFloat(formattedFloatVal, intSize, valSize, masterStatsItems_[i].double_value,
- FALSE, TRUE);
- sprintf(statsBuf_, "%-25s%s", "Estimated Accessed Rows",
- formattedFloatVal);
+ sprintf(statsBuf_, "%-25s%.6g", "Estimated Accessed Rows", masterStatsItems_[i].double_value);
break;
case SQLSTATS_EST_ROWS_USED:
- FormatFloat(formattedFloatVal, intSize, valSize, masterStatsItems_[i].double_value,
- FALSE, TRUE);
- sprintf(statsBuf_, "%-25s%s", "Estimated Used Rows", formattedFloatVal);
+ sprintf(statsBuf_, "%-25s%.6g", "Estimated Used Rows", masterStatsItems_[i].double_value);
break;
case SQLSTATS_PARENT_QUERY_ID:
masterStatsItems_[i].str_value[masterStatsItems_[i].str_ret_len] = '\0';
@@ -2959,13 +2919,15 @@ short ExExeUtilGetRTSStatisticsTcb::work()
}
break;
case SQLSTATS_EST_ROWS_ACCESSED:
+ sprintf(formattedFloatVal, "%.6g", statsItems[i].double_value);
if (singleLineFormat_)
- sprintf(&statsBuf_[strlen(statsBuf_)], "%15.0f", statsItems[i].double_value);
+ sprintf(&statsBuf_[strlen(statsBuf_)], "%15s", formattedFloatVal);
else
sprintf(statsBuf_, "%15s", formattedFloatVal);
break;
case SQLSTATS_EST_ROWS_USED:
- sprintf(&statsBuf_[strlen(statsBuf_)], "%15.0f", statsItems[i].double_value);
+ sprintf(formattedFloatVal, "%.6g", statsItems[i].double_value);
+ sprintf(&statsBuf_[strlen(statsBuf_)], "%15s", formattedFloatVal);
break;
case SQLSTATS_ACT_ROWS_ACCESSED:
sprintf(Int64Val, "%ld", statsItems[i].int64_value);
@@ -3234,8 +3196,8 @@ short ExExeUtilGetRTSStatisticsTcb::work()
sprintf(&statsBuf_[strlen(statsBuf_)], "%20s", Int64Val);
break;
case SQLSTATS_BMO_EST_MEMORY:
- sprintf(formattedFloatVal, "%0.2lf", bmoStatsItems_[i].double_value);
- str_sprintf(&statsBuf_[strlen(statsBuf_)], "%-20s", formattedFloatVal);
+ sprintf(formattedFloatVal, "%.6g", bmoStatsItems_[i].double_value);
+ str_sprintf(&statsBuf_[strlen(statsBuf_)], "%20s", formattedFloatVal);
break;
case SQLSTATS_BMO_SPACE_BUFFER_SIZE:
sprintf(Int64Val, "%ld", bmoStatsItems_[i].int64_value);
[3/3] incubator-trafodion git commit: Merge PR 1332 Fixes for
[TRAFODION-2836] and [TRAFODION-2420]
Posted by se...@apache.org.
Merge PR 1332 Fixes for [TRAFODION-2836] and [TRAFODION-2420]
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/74953778
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/74953778
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/74953778
Branch: refs/heads/master
Commit: 7495377831d1cb39343014df9e5f72a99a9c00d7
Parents: 160785a 68764fc
Author: selvaganesang <se...@apache.org>
Authored: Tue Dec 12 19:17:21 2017 +0000
Committer: selvaganesang <se...@apache.org>
Committed: Tue Dec 12 19:17:21 2017 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 2 -
core/sql/executor/ExExeUtilGetStats.cpp | 64 ++++++--------------------
core/sql/executor/JavaObjectInterface.cpp | 38 ++++++++++++---
3 files changed, 45 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-trafodion git commit: [TRAFODION-2836] Enable
Trafodion SQL processes to configure garbage collector in its embedded JVM
Posted by se...@apache.org.
[TRAFODION-2836] Enable Trafodion SQL processes to configure garbage collector
in its embedded JVM
Provided a way to configure the garbage collector when the JVM is created
in EsgynDB processes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/68764fcf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/68764fcf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/68764fcf
Branch: refs/heads/master
Commit: 68764fcf109e0020c4714567fcc3693e10f1c43c
Parents: 1a3dc67
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Dec 8 23:14:05 2017 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Fri Dec 8 23:14:05 2017 +0000
----------------------------------------------------------------------
core/sql/executor/JavaObjectInterface.cpp | 38 ++++++++++++++++++++++----
1 file changed, 32 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/68764fcf/core/sql/executor/JavaObjectInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/JavaObjectInterface.cpp b/core/sql/executor/JavaObjectInterface.cpp
index ff9f352..2f5b2c7 100644
--- a/core/sql/executor/JavaObjectInterface.cpp
+++ b/core/sql/executor/JavaObjectInterface.cpp
@@ -26,13 +26,14 @@
#include "Globals.h"
#include "ComUser.h"
#include "LmJavaOptions.h"
+#include "ex_ex.h"
-// Changed the default to 256 to limit java heap size used by SQL processes.
-// Keep this define in sync with udrserv/udrserv.cpp
#define DEFAULT_JVM_MAX_HEAP_SIZE 256
#define DEFAULT_COMPRESSED_CLASSSPACE_SIZE 128
#define DEFAULT_MAX_METASPACE_SIZE 128
#define TRAF_DEFAULT_JNIHANDLE_CAPACITY 32
+
+
// ===========================================================================
// ===== Class JavaObjectInterface
// ===========================================================================
@@ -75,6 +76,8 @@ char* JavaObjectInterface::getErrorText(JOI_RetCode errEnum)
//////////////////////////////////////////////////////////////////////////////
JavaObjectInterface::~JavaObjectInterface()
{
+ if (jenv_ == NULL)
+ return;
if ((long)javaObj_ != -1)
jenv_->DeleteGlobalRef(javaObj_);
javaObj_ = NULL;
@@ -136,6 +139,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
{
classPathArg = buildClassPath();
jvm_options[numJVMOptions].optionString = classPathArg;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG, "Using classpath: %s",
jvm_options[numJVMOptions].optionString);
numJVMOptions++;
@@ -145,7 +149,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
{
int maxHeapEnvvarMB = DEFAULT_JVM_MAX_HEAP_SIZE;
const char *maxHeapSizeStr = getenv("JVM_MAX_HEAP_SIZE_MB");
- if (maxHeapSizeStr)
+ if (maxHeapSizeStr != NULL)
{
maxHeapEnvvarMB = atoi(maxHeapSizeStr);
if (maxHeapEnvvarMB <= 0)
@@ -155,17 +159,30 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
snprintf(maxHeapOptions, sizeof(maxHeapOptions),
"-Xmx%dm", maxHeapEnvvarMB);
jvm_options[numJVMOptions].optionString = maxHeapOptions;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"Max heap option: %s",
jvm_options[numJVMOptions].optionString);
numJVMOptions++;
}
+ const char *jvmGC = getenv("JVM_GC_OPTION");
+ if (jvmGC != NULL)
+ {
+ jvm_options[numJVMOptions].optionString = (char *)jvmGC;
+ jvm_options[numJVMOptions].extraInfo = NULL;
+ QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
+ "GC Option: %s",
+ jvm_options[numJVMOptions].optionString);
+ numJVMOptions++;
+ }
+
+
if (!isDefinedInOptions(options, "-XX:CompressedClassSpaceSize="))
{
int compressedClassSpaceSize = 0;
const char *compressedClassSpaceSizeStr = getenv("JVM_COMPRESSED_CLASS_SPACE_SIZE");
- if (compressedClassSpaceSizeStr)
+ if (compressedClassSpaceSizeStr != NULL)
compressedClassSpaceSize = atoi(compressedClassSpaceSizeStr);
if (compressedClassSpaceSize <= 0)
compressedClassSpaceSize = DEFAULT_COMPRESSED_CLASSSPACE_SIZE;
@@ -173,6 +190,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
sizeof(compressedClassSpaceSizeOptions),
"-XX:CompressedClassSpaceSize=%dm", compressedClassSpaceSize);
jvm_options[numJVMOptions].optionString = compressedClassSpaceSizeOptions;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"CompressedClassSpaceSize: %s",
jvm_options[numJVMOptions].optionString);
@@ -183,13 +201,14 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
{
int maxMetaspaceSize = 0;
const char *maxMetaspaceSizeStr = getenv("JVM_MAX_METASPACE_SIZE");
- if (maxMetaspaceSizeStr)
+ if (maxMetaspaceSizeStr != NULL)
maxMetaspaceSize = atoi(maxMetaspaceSizeStr);
if (maxMetaspaceSize <= 0)
maxMetaspaceSize = DEFAULT_MAX_METASPACE_SIZE;
snprintf(maxMetaspaceSizeOptions, sizeof(maxMetaspaceSizeOptions),
"-XX:MaxMetaspaceSize=%dm", maxMetaspaceSize);
jvm_options[numJVMOptions].optionString = maxMetaspaceSizeOptions;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"MaxMetaspaceSize: %s",
jvm_options[numJVMOptions].optionString);
@@ -199,7 +218,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
if (!isDefinedInOptions(options, "-Xms"))
{
const char *initHeapSizeStr = getenv("JVM_INIT_HEAP_SIZE_MB");
- if (initHeapSizeStr)
+ if (initHeapSizeStr != NULL)
{
const int initHeapEnvvarMB = atoi(initHeapSizeStr);
if (initHeapEnvvarMB > 0)
@@ -207,6 +226,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
snprintf(initHeapOptions, sizeof(initHeapOptions),
"-Xms%dm", initHeapEnvvarMB);
jvm_options[numJVMOptions].optionString = initHeapOptions;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"Init heap option: %s",
jvm_options[numJVMOptions].optionString);
@@ -243,6 +263,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
else
strcat(debugOptions, ",suspend=n");
jvm_options[numJVMOptions].optionString = debugOptions;
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_WARN,
"Debugging JVM with options: %s",
jvm_options[numJVMOptions].optionString);
@@ -253,6 +274,7 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
if (!isDefinedInOptions(options, oomOption))
{
jvm_options[numJVMOptions].optionString = (char *)oomOption;
+ jvm_options[numJVMOptions].extraInfo = NULL;
numJVMOptions++;
}
@@ -268,14 +290,18 @@ int JavaObjectInterface::createJVM(LmJavaOptions *options)
strcat(oomDumpDir, mySqRoot);
strcat(oomDumpDir, "/logs");
jvm_options[numJVMOptions].optionString = (char *)oomDumpDir;
+ jvm_options[numJVMOptions].extraInfo = NULL;
numJVMOptions++;
}
}
+ ex_assert((numJVMOptions < MAX_NO_JVM_OPTIONS), "Buffer overflow in JVM options");
+
if (options)
for (CollIndex o=0; o<options->entries(); o++)
{
jvm_options[numJVMOptions].optionString = (char *) options->getOption(o);
+ jvm_options[numJVMOptions].extraInfo = NULL;
QRLogger::log(CAT_SQL_HDFS_JNI_TOP, LL_DEBUG,
"Option passed to JavaObjectInterface::createJVM(): %s",
jvm_options[numJVMOptions].optionString);