You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by db...@apache.org on 2017/03/31 18:51:05 UTC
[1/2] incubator-trafodion git commit: [TRAFODION-2376] Improve UPDATE
STATS performance on varchar columns
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 8691d783d -> 29307b4c0
[TRAFODION-2376] Improve UPDATE STATS performance on varchar columns
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3366fdba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3366fdba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3366fdba
Branch: refs/heads/master
Commit: 3366fdba1b9d52e7d04d21ee33f92698089cdb36
Parents: 6155ff1
Author: Dave Birdsall <db...@apache.org>
Authored: Tue Mar 28 17:16:00 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Tue Mar 28 17:16:00 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/analyzeULOG.py | 177 ++++++++++++++
core/sql/common/wstr.cpp | 41 ++++
core/sql/common/wstr.h | 11 +
core/sql/sqlcomp/DefaultConstants.h | 1 +
core/sql/sqlcomp/nadefaults.cpp | 1 +
core/sql/ustat/hs_cli.cpp | 7 +-
core/sql/ustat/hs_globals.cpp | 408 ++++++++++++++++++++++++-------
core/sql/ustat/hs_globals.h | 60 ++++-
core/sql/ustat/hs_read.cpp | 12 +-
9 files changed, 626 insertions(+), 92 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sqf/sql/scripts/analyzeULOG.py
----------------------------------------------------------------------
diff --git a/core/sqf/sql/scripts/analyzeULOG.py b/core/sqf/sql/scripts/analyzeULOG.py
new file mode 100644
index 0000000..4e17626
--- /dev/null
+++ b/core/sqf/sql/scripts/analyzeULOG.py
@@ -0,0 +1,177 @@
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+#
+# This script analyzes data from a ULOG. It looks for timer
+# events, and pulls these out, generating SQL INSERT
+# statements with their data. This data can then be loaded
+# into a SQL table for further analysis.
+#
+# The shape of the table assumed by the script is as follows:
+#
+# create table ulog_data
+# ( l1 int not null,
+# l2 int not null,
+# l3 int not null,
+# l4 int not null,
+# l5 int not null,
+# l6 int not null,
+# l7 int not null,
+# l8 int not null,
+# l9 int not null,
+# l10 int not null,
+# task varchar(80) not null,
+# elapsed char(12) not null,
+# primary key (l1,l2,l3,l4,l5,l6,l7,l8,l9,l10) );
+#
+# The idea here is to capture the nested structure of the ULOG
+# timers. L1 represents the top-most level and is incremented
+# for each UPDATE STATISTICS statement. L2 is the set of timers
+# within the scope of an L1 timer, and so on.
+#
+# By querying this ulog_data, one can get an idea of where
+# UPDATE STATISTICS spends its time, and how expensive each
+# task is.
+#
+import os
+import sys
+import subprocess
+import sets
+import datetime
+import argparse # requires Python 2.7
+
+# An object of class BeginEndInterval is created for each timer
+# found in the ULOG.
+
+class BeginEndInterval:
+ #
+
+ def __init__(self, desc, level, previousItemNumber):
+ #print "On entry to ctor, level = " + str(level) + ", prevIN = " + str(previousItemNumber)
+ self.description = desc
+ self.elapsedTime = None
+ self.itemNumber = list(previousItemNumber)
+ if len(previousItemNumber) <= level:
+ self.itemNumber.append(1)
+ else:
+ lastIndex = len(previousItemNumber)-1
+ self.itemNumber[lastIndex] = self.itemNumber[lastIndex]+1
+ #print "Exiting ctor, Resulting in " + str(self.itemNumber)
+
+ def setEnd(self, desc, elapsed):
+ assert desc == self.description
+ self.elapsedTime = elapsed
+
+ def generateData(self):
+ text = "INSERT INTO ULOG_DATA VALUES("
+ for i in range(10):
+ if i < len(self.itemNumber):
+ text = text + str(self.itemNumber[i]) + ","
+ else:
+ text = text + "0,"
+ text = text + "'" + self.description + "','" + self.elapsedTime + "');"
+ print text
+
+
+class ParseULOG:
+ #
+
+ def __init__(self, ULOGFileName):
+ self.ULOGFileName = ULOGFileName
+ self.beginEndIntervals = []
+
+
+ def parseULOG(self):
+ #
+ # Begin/End entries look like this:
+ #
+ # [Tue Mar 21 21:02:27 2017] :| BEGIN Allocate storage for columns
+ # [Tue Mar 21 21:02:30 2017] :| END Allocate storage for columns elapsed time (00:00:02.666)
+ #
+ # The number of vertical bars varies and indicates levels of nesting.
+ # For now we ignore the timestamp, and just pick out the description
+ # and the elapsed time.
+ #
+
+ try:
+ f = open(self.ULOGFileName)
+ previousItemNumber = []
+ messageNumberStr = None
+ messageText = None
+ for line in f:
+ #originalLine = line
+ line = line.rstrip('\n') # get rid of trailing return character
+ index = line.find('] ')
+ if index >= 0:
+ line = line[index+2:] # strip off leading timestamp part
+ if line.startswith(':'):
+ line = line[1:] # strip off colon
+ level = 0
+ while line.startswith('| '):
+ level = level + 1
+ line = line[3:]
+ if line.startswith('BEGIN '):
+ description = line[6:] # strip off BEGIN; description is what's left
+ # create a BeginEnd interval and add to stack
+ beginEndInterval = BeginEndInterval(description,level,previousItemNumber)
+ previousItemNumber = list(beginEndInterval.itemNumber)
+ self.beginEndIntervals.append(beginEndInterval)
+ elif line.startswith('END '):
+ index = line.find(' elapsed time (')
+ if index >= 0:
+ description = line[6:index]
+ elapsedTime = line[index+15:]
+ elapsedTime = elapsedTime.rstrip(')')
+ # pop the last BeginEnd interval from the stack
+ beginEndInterval = self.beginEndIntervals.pop()
+ beginEndInterval.setEnd(description,elapsedTime)
+ beginEndInterval.generateData()
+ previousItemNumber = list(beginEndInterval.itemNumber)
+ del beginEndInterval
+ f.close()
+
+ except IOError as detail:
+ print "Could not open " + self.ULOGFileName
+ print detail
+
+
+
+
+
+# beginning of main
+
+
+# process command line arguments
+
+parser = argparse.ArgumentParser(
+ description='This script parses out interesting data from a ULOG.')
+parser.add_argument("ULOGFileName", help='The name of the ULOG file you wish to parse.')
+
+args = parser.parse_args() # exits and prints help if args are incorrect
+
+exitCode = 0
+
+ULOGparser = ParseULOG(args.ULOGFileName)
+
+ULOGparser.parseULOG()
+
+exit(exitCode)
+
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/common/wstr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/wstr.cpp b/core/sql/common/wstr.cpp
index e5bd656..9c99a36 100644
--- a/core/sql/common/wstr.cpp
+++ b/core/sql/common/wstr.cpp
@@ -64,6 +64,47 @@ int iswspace (NAWchar wc)
}
*/
+// Compares two Wchar strings using SQL blank-padding semantics.
+// Returns +ve if the first string is greater, 0 if equal, -ve if
+// the second string is greater.
+Int32 compareWcharWithBlankPadding(const NAWchar *wstr1, UInt32 len1,
+ const NAWchar *wstr2, UInt32 len2)
+{
+ UInt32 shorterLen = (len1 < len2 ? len1 : len2);
+ Int32 rc = memcmp(wstr1,wstr2,shorterLen*sizeof(NAWchar));
+ if (rc == 0)
+ {
+ if (len1 < len2)
+ {
+ // compare the rest of wstr2 with blanks
+ while ((shorterLen < len2) && (wstr2[shorterLen] == L' '))
+ shorterLen++;
+
+ if (shorterLen < len2)
+ {
+ if (wstr2[shorterLen] > L' ')
+ rc = -1;
+ else
+ rc = 1;
+ }
+ }
+ else if (len1 > len2)
+ {
+ // compare the rest of wstr1 with blanks
+ while ((shorterLen < len1) && (wstr1[shorterLen] == L' '))
+ shorterLen++;
+
+ if (shorterLen < len1)
+ {
+ if (wstr1[shorterLen] > L' ')
+ rc = 1;
+ else
+ rc = -1;
+ }
+ }
+ }
+ return rc;
+}
// Do a char-by-char comparison (including nulls) up to the length of the
// shorter string or the first difference. If the strings are equal up to the
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/common/wstr.h
----------------------------------------------------------------------
diff --git a/core/sql/common/wstr.h b/core/sql/common/wstr.h
index 590e5b0..a7f9d81 100644
--- a/core/sql/common/wstr.h
+++ b/core/sql/common/wstr.h
@@ -45,6 +45,17 @@
#include "unicode_char_set.h"
// -----------------------------------------------------------------------
+// Compare w-strings <left> and <right> (using unsigned comparison and
+// SQL blank-padding semantics)
+// for <length> characters.
+// Return a negative value if left < right,
+// return 0 if left == right,
+// return a positive value if left > right.
+// -----------------------------------------------------------------------
+Int32 compareWcharWithBlankPadding(const NAWchar *wstr1, UInt32 len1,
+ const NAWchar *wstr2, UInt32 len2);
+
+// -----------------------------------------------------------------------
// Compare strings <left> and <right> (using unsigned comparison).
// for <length> characters.
// Return a negative value if left < right,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 96c8f1e..941de30 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -677,6 +677,7 @@ enum DefaultConstants
// salted table when ON EVERY KEY or ON EVERY COLUMN is specified.
USTAT_ATTEMPT_ESP_PARALLELISM, // use parallel plans for reading columns to form histograms
USTAT_CHECK_HIST_ACCURACY, // After stats collection, examine full table and calculate accuray of hists
+ USTAT_COMPACT_VARCHARS, // For internal sort, store only the actual # chars used in each value
USTAT_CLUSTER_SAMPLE_BLOCKS, // number of blocks for cluster sampling
USTAT_ESTIMATE_HBASE_ROW_COUNT, // If ON, estimate row count of HBase table instead of count(*), subject
// to USTAT_MIN_ESTIMATE_FOR_ROWCOUNT setting)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 0814a15..3293e2b 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -3528,6 +3528,7 @@ XDDkwd__(SUBQUERY_UNNESTING, "ON"),
DDkwd__(USTAT_COLLECT_MC_SKEW_VALUES, "OFF"),
+ DDkwd__(USTAT_COMPACT_VARCHARS, "OFF"), // If on, internal sort does not pad out varchars
DD_____(USTAT_CQDS_ALLOWED_FOR_SPAWNED_COMPILERS, ""), // list of CQDs that can be pushed to seconday compilers
// CQDs are delimited by ","
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/ustat/hs_cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_cli.cpp b/core/sql/ustat/hs_cli.cpp
index a028658..873a1a0 100644
--- a/core/sql/ustat/hs_cli.cpp
+++ b/core/sql/ustat/hs_cli.cpp
@@ -2968,7 +2968,12 @@ Lng32 HSCursor::setRowsetPointers(HSColGroupStruct *group, Lng32 maxRows)
// Character data is written into a different buffer, and the data buffer
// will consist of pointers to the char values.
if (DFS2REC::isAnyCharacter(group->ISdatatype))
- rowset_fields_[j].var_ptr = (void *)group->strNextData;
+ {
+ if (DFS2REC::isSQLVarChar(group->ISdatatype) && group->isCompacted())
+ rowset_fields_[j].var_ptr = (void *)group->varcharFetchBuffer;
+ else
+ rowset_fields_[j].var_ptr = (void *)group->strNextData;
+ }
else
rowset_fields_[j].var_ptr = (void *)group->nextData;
j++;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index 91c7bbf..243e3d0 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -53,6 +53,7 @@
#include "hs_faststats.h"
#include "ComCextdecs.h"
#include "NAString.h"
+#include "wstr.h"
#include "Collections.h"
#include "NumericType.h"
#include "exp_datetime.h"
@@ -1344,6 +1345,13 @@ void ISVarChar::fail(const char* opName, Lng32 line)
// and positive value if greater.
Int32 ISVarChar::compare(const ISVarChar &rhs)
{
+ Int32 result;
+ Int16 lhsLen = *(short*)content;
+ Int16 rhsLen = *(short*)rhs.content;
+ Int16 minLen = MINOF(lhsLen, rhsLen);
+ Int16 diffLen;
+ char* diffPtr;
+
// Note that case insensitive is not supported with non-binary collation.
if (CollationInfo::isSystemCollation(colCollation))
return Collated_cmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
@@ -1354,11 +1362,34 @@ Int32 ISVarChar::compare(const ISVarChar &rhs)
// UCS2 cols not supported in MODE_SPECIAL_1 and do not support case insensitivity.
if (!caseInsensitive) {
if (charset != CharInfo::UNICODE)
- return memcmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
+ {
+ result = memcmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
rhs.content+VARCHAR_LEN_FIELD_IN_BYTES,
- MAXOF(*((short*)content), *((short*)rhs.content)));
+ minLen);
+ if (result != 0 || lhsLen == rhsLen)
+ return result;
+ else
+ {
+ if (minLen == lhsLen)
+ {
+ diffPtr = rhs.content + VARCHAR_LEN_FIELD_IN_BYTES + minLen;
+ diffLen = rhsLen - minLen;
+ }
+ else
+ {
+ diffPtr = content + VARCHAR_LEN_FIELD_IN_BYTES + minLen;
+ diffLen = lhsLen - minLen;
+ }
+ for (int i = 0; i < diffLen; i++)
+ {
+ if (*diffPtr++ != ' ')
+ return (minLen == lhsLen ? -1 : 1);
+ }
+ return 0;
+ }
+ }
else
- return na_wcsnncmp((const wchar_t*)(content+VARCHAR_LEN_FIELD_IN_BYTES),
+ return compareWcharWithBlankPadding((const wchar_t*)(content+VARCHAR_LEN_FIELD_IN_BYTES),
*((short*)content) / sizeof(NAWchar),
(const wchar_t*)(rhs.content+VARCHAR_LEN_FIELD_IN_BYTES),
*((short*)rhs.content) / sizeof(NAWchar));
@@ -1371,22 +1402,7 @@ Int32 ISVarChar::compare(const ISVarChar &rhs)
Int32 ISVarChar::operator==(const ISVarChar &rhs)
{
- // Note that case insensitive is not supported with non-binary collation.
- if (CollationInfo::isSystemCollation(colCollation))
- return (Collated_cmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
- rhs.content+VARCHAR_LEN_FIELD_IN_BYTES,
- MAXOF(*((short*)content), *((short*)rhs.content)),
- colCollation, sortBuffer1, sortBuffer2 ) == 0);
-
- // UCS2 cols not supported in MODE_SPECIAL_1 and do not support case insensitivity.
- if (!caseInsensitive)
- return !memcmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
- rhs.content+VARCHAR_LEN_FIELD_IN_BYTES,
- MAXOF(*((short*)content), *((short*)rhs.content)));
- else
- return !hs_strncasecmp(content+VARCHAR_LEN_FIELD_IN_BYTES,
- rhs.content+VARCHAR_LEN_FIELD_IN_BYTES,
- MAXOF(*((short*)content), *((short*)rhs.content)));
+ return !compare(rhs); // returns 1 if equal, 0 if not
}
void IUSFixedChar::operator=(const HSDataBuffer& buff)
@@ -1457,14 +1473,18 @@ HSColGroupStruct::HSColGroupStruct()
: colSet(STMTHEAP), colCount(0), clistr(new(STMTHEAP) NAString(STMTHEAP)),
oldHistid(0), newHistid(0), colNames(new(STMTHEAP) NAString(STMTHEAP)),
groupHist(NULL), next(NULL), prev(NULL), state(UNPROCESSED),
- memNeeded(0), data(NULL), nextData(NULL), strData(NULL), strNextData(NULL),
+ memNeeded(0), strMemAllocated(0),
+ data(NULL), nextData(NULL), strData(NULL), strNextData(NULL),
strDataConsecutive(TRUE), // only becomes false if data sets merged for IUS
+ varcharFetchBuffer(NULL),
mcis_data(NULL), mcis_nextData(NULL), mcs_usingme(0), //for MC
nullIndics(NULL), nullCount(0), mcis_rowsRead(0),
- ISdatatype(-1), ISlength(-1), ISprecision(-1), ISscale(-1),
+ eligibleForVarCharCompaction(FALSE),
+ ISdatatype(-1), ISlength(-1), ISvcLenUsed(-1), ISprecision(-1), ISscale(-1),
ISSelectExpn(STMTHEAP), prevRowCount(0), prevUEC(0),
reason(HS_REASON_UNKNOWN), newReason(HS_REASON_MANUAL),
- colSecs(0), coeffOfVar(0), avgVarCharSize(-1), skewedValuesCollected(FALSE),
+ colSecs(0), coeffOfVar(0), oldAvgVarCharSize(-1), rowsRead(0), sumSize(0),
+ avgVarCharSize(-1), skewedValuesCollected(FALSE),
mcis_nullIndBitMap(NULL), mcis_colsUsedMap(NULL),
mcis_colsMissingMap(NULL), mcis_memFreed(FALSE),
mcis_totalMCmemNeeded(0), mcis_groupHead(TRUE), mcis_next(NULL), mcis_readAsIs (FALSE),
@@ -1487,6 +1507,88 @@ HSColGroupStruct::~HSColGroupStruct()
freeISMemory();
}
+/**
+ * Sets the length of the IS type of the column, and the estimated average
+ * length if the mapped type is varchar and compacted varchars are in use.
+ *
+ * @param len Natural length of the column type as represented for IS.
+ * @param maxCharColumnLengthInBytes Maximum length character string limit
+ * imposed by UPDATE STATS
+ */
+void HSColGroupStruct::setISlength(Lng32 len, Lng32 maxCharColumnLengthInBytes)
+{
+ ISlength = MINOF(len, maxCharColumnLengthInBytes);
+ if (!DFS2REC::isAnyVarChar(ISdatatype))
+ return;
+
+ if (eligibleForVarCharCompaction)
+ {
+ // If average varchar size is known from older histograms
+ // use that; otherwise use a rule of thumb estimate.
+
+ if (oldAvgVarCharSize >= 1)
+ ISvcLenUsed = oldAvgVarCharSize + 4; // + 4 to allow a little growth
+ else
+ {
+ // In the absence of older histograms, assume the average
+ // length is about one half the maximum length. (After all,
+ // the user presumably chose varchar to save some space.)
+ // Note: This code path can only be taken on the first call
+ // to this method. Later calls happen only when we overran
+ // buffer space, but in that case oldAvgVarCharSize will
+ // have been calculated.
+ double ruleOfThumbEstimate = len/2;
+ if (ruleOfThumbEstimate < 4)
+ ruleOfThumbEstimate = 4;
+ ISvcLenUsed = ruleOfThumbEstimate;
+ }
+
+ if (ISvcLenUsed > ISlength)
+ ISvcLenUsed = ISlength; // don't allow it to exceed the actual size!
+
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "Considering compaction on varchar column %s:", colNames->data());
+ LM->Log(LM->msg);
+ sprintf(LM->msg, " Declared len: %d, estimated avg len: %d", ISlength, ISvcLenUsed);
+ LM->Log(LM->msg);
+ sprintf(LM->msg, " Compaction%schosen", ISvcLenUsed == len ? " not " : " ");
+ LM->Log(LM->msg);
+ }
+ }
+ else
+ ISvcLenUsed = ISlength;
+}
+
+/**
+ * Determines the number of bytes to allocate for strData, the buffer holding
+ * all the data for a char/varchar column with internal sort. The overall memory
+ * requirement for the column has already been calculated. Here, we just need to
+ * determine how much of it is for the data buffer. The other parts are the array
+ * of objects (ISFixedChar or ISVarChar) that reference the content, and for
+ * a compacted varchar, the buffer that the uncompacted varchar data is read into.
+ *
+ * @param rows Number of rows being retrieved to calculate stats on.
+ * @return Number of bytes to allocate to hold the char or varchar content.
+ */
+size_t HSColGroupStruct::strDataMemNeeded(Int64 rows)
+{
+ size_t result = memNeeded;
+ HS_ASSERT(DFS2REC::isAnyCharacter(ISdatatype));
+ if (DFS2REC::isAnyVarChar(ISdatatype))
+ {
+ result -= (rows * sizeof(ISVarChar)); // deduct space for ptrs to content
+ if (isCompacted())
+ result -= (inflatedVarcharContentSize() * MAX_ROWSET); // deduct pre-compaction fetch buffer
+ }
+ else
+ result -= (rows * sizeof(ISFixedChar));
+
+ return result;
+}
+
+
// Allocates memory necessary for internal sort for the group. If an allocation
// failure occurs, free any memory already allocated for the current column and
// exit.
@@ -1509,8 +1611,6 @@ NABoolean HSColGroupStruct::allocateISMemory(Int64 rows,
NABoolean recalcMemNeeded)
{
HSLogMan *LM = HSLogMan::Instance();
- Int64 uvCharCount;
- size_t strMemNeeded;
NAWchar* wptr;
NABoolean allAllocated = TRUE;
@@ -1559,7 +1659,11 @@ NABoolean HSColGroupStruct::allocateISMemory(Int64 rows,
//
// memNeeded includes length for ptrs; subtract it from amount of space
// to allocate for the strings themselves.
- strMemNeeded = memNeeded - (size_t)(sizeof(void*) * rows);
+ size_t strMemNeeded = strDataMemNeeded(rows);
+ // round up to next multiple of sizeof(short)
+ strMemNeeded = sizeof(short) * ( (strMemNeeded + sizeof(short) - 1) / sizeof(short) );
+ strMemAllocated = strMemNeeded; // remember for overrun checking
+
if (DFS2REC::isAnyVarChar(ISdatatype))
{
@@ -1570,15 +1674,31 @@ NABoolean HSColGroupStruct::allocateISMemory(Int64 rows,
if (!strData)
throw ISMemAllocException();
- if (ISdatatype == REC_BYTE_V_DOUBLE)
- {
- uvCharCount = strMemNeeded / sizeof(NAWchar);
- wptr = (NAWchar*)strData + uvCharCount - 1;
- while (uvCharCount--)
- *wptr-- = L' ';
- }
+ // Unless varchar values are compacted after being read, blank out the
+ // entire varchar allocation to allow simple blank-padded comparison
+ // of different-length strings.
+ if (isCompacted())
+ {
+ size_t fetchMemNeeded = (inflatedVarcharContentSize() * MAX_ROWSET);
+ fetchMemNeeded = sizeof(short) *
+ ( (fetchMemNeeded + sizeof(short) - 1) / sizeof(short) );
+ varcharFetchBuffer =
+ (char*)(newBuiltinArr(short, fetchMemNeeded / sizeof(short)));
+ if (!varcharFetchBuffer)
+ throw ISMemAllocException();
+ }
else
- memset(strData, ' ', strMemNeeded);
+ {
+ if (ISdatatype == REC_BYTE_V_DOUBLE)
+ {
+ Int64 uvCharCount = strMemNeeded / sizeof(NAWchar);
+ wptr = (NAWchar*)strData + uvCharCount - 1;
+ while (uvCharCount--)
+ *wptr-- = L' ';
+ }
+ else
+ memset(strData, ' ', strMemNeeded);
+ }
}
#ifdef _TEST_ALLOC_FAILURE
data = newObjArrX(ISVarChar, rows, allocCount++);
@@ -1663,6 +1783,11 @@ void HSColGroupStruct::freeISMemory(NABoolean freeStrData, NABoolean freeMCData)
{
NADELETEBASIC((short*)strData, STMTHEAP);
strData = NULL;
+ if (isCompacted())
+ {
+ NADELETEBASIC((short*)varcharFetchBuffer, STMTHEAP);
+ varcharFetchBuffer = NULL;
+ }
}
}
else
@@ -4979,9 +5104,8 @@ static void mapInternalSortTypes(HSColGroupStruct *groupList, NABoolean forHive
{
HSGlobalsClass *hs_globals = GetHSContext();
group->ISdatatype = col.datatype;
- group->ISlength = col.length;
- if (group->ISlength > hs_globals->maxCharColumnLengthInBytes)
- group->ISlength = hs_globals->maxCharColumnLengthInBytes;
+ //group->ISlength = col.length;
+ group->setISlength(col.length,hs_globals->maxCharColumnLengthInBytes);
group->ISprecision = col.precision;
group->ISscale = col.scale;
// the method below handles adding SUBSTRING for over-size char/varchars
@@ -5113,14 +5237,7 @@ void HSGlobalsClass::getMemoryRequirementsForOneGroup(HSColGroupStruct* group, I
case REC_BYTE_V_ASCII:
case REC_BYTE_V_DOUBLE:
- // Length is max bytes. Add size of length field, and alignment byte
- // if max length is odd. Also add for object that references the string,
- // which is stored in a separate array.
- elementSize = group->ISlength
- + VARCHAR_LEN_FIELD_IN_BYTES
- + (group->ISlength%2)
- + sizeof(ISVarChar);
- //cumuVarCharSize += elementSize;
+ elementSize = group->varcharContentSize() + sizeof(ISVarChar);
break;
default:
@@ -5145,7 +5262,11 @@ void HSGlobalsClass::getMemoryRequirementsForOneGroup(HSColGroupStruct* group, I
}
Int64 i64MemNeeded = rows * elementSize;
- group->memNeeded = (i64MemNeeded <= UINT_MAX ? (size_t)(rows * elementSize) : 0);
+ if (group->isCompacted()) // varchar only
+ {
+ i64MemNeeded += (MAX_ROWSET * group->inflatedVarcharContentSize());
+ }
+ group->memNeeded = (i64MemNeeded <= UINT_MAX ? (size_t)i64MemNeeded : 0);
if (LM->LogNeeded())
{
if (group->memNeeded == 0)
@@ -5386,6 +5507,39 @@ Lng32 HSGlobalsClass::CollectStatistics()
}
else // internal sort is enabled
{
+ // Figure out which groups are eligible for varchar compaction:
+ // A varchar column is eligible if CQD USTAT_COMPACT_VARCHARS is 'ON'
+ // and that column is not referenced by any multi-column group.
+ // The reason for the latter condition is we want to avoid the
+ // possibility of doing a second full table sample scan in the event
+ // that we attempt to do multi-column histograms in-memory, and
+ // we underestimate the memory needed for internal sort.
+
+ NABoolean varcharCompactionFeasible = FALSE;
+ if (CmpCommon::getDefault(USTAT_COMPACT_VARCHARS) == DF_ON)
+ {
+ NABitVector * refdColumns = new (STMTHEAP) NABitVector (STMTHEAP);
+ for (HSColGroupStruct * mcgrp = multiGroup; mcgrp; mcgrp = mcgrp->next)
+ {
+ for (Int32 i = 0; i < mcgrp->colCount; i++)
+ {
+ HSColumnStruct *c = &mcgrp->colSet[i];
+ refdColumns->setBit(c->colnum);
+ }
+ }
+
+ for (HSColGroupStruct * sgrp = singleGroup; sgrp; sgrp = sgrp->next)
+ {
+ if (!refdColumns->contains(sgrp->colSet[0].colnum))
+ {
+ sgrp->eligibleForVarCharCompaction = TRUE;
+ varcharCompactionFeasible = TRUE;
+ }
+ }
+
+ delete refdColumns;
+ }
+
// Get percentage of available memory to recommend. If an allocation
// for memory for a column fails, this percentage will be reduced
// for subsequent selection of column batches.
@@ -5394,6 +5548,11 @@ Lng32 HSGlobalsClass::CollectStatistics()
NABoolean trySampleTableBypassForIS = useSampling && externalSampleTable == FALSE;
+ // If we are considering varchar compaction and IS, get previous histogram
+ // information here.
+ if (varcharCompactionFeasible)
+ getPreviousUECRatios(singleGroup);
+
mapInternalSortTypes(singleGroup);
Int64 maxRowsToRead = getInternalSortMemoryRequirements(TRUE);
@@ -5470,10 +5629,10 @@ Lng32 HSGlobalsClass::CollectStatistics()
sgroup = NULL;
}
- if (internalSortWhenBetter)
+ // If we need UEC ratios info and we haven't already read it, do so now
+ if (internalSortWhenBetter && !varcharCompactionFeasible)
getPreviousUECRatios(singleGroup); // used to decide when to use IS
-
if ( performISForMC() )
{
@@ -10463,7 +10622,8 @@ Lng32 HSGlobalsClass::processInternalSortNulls(Lng32 rowsRead, HSColGroupStruct
NABoolean computeVarCharSize = FALSE;
NABoolean maxLongLimit = FALSE;
short *nullInd = NULL;
- Int32 vcLen;
+ Int32 vcInflatedLen, vcCompactLen;
+ char* inflatedDataPtr;
char *dataPtr, errtxt[100]={0};
Int32 i;
Lng32 retcode=0;
@@ -10471,7 +10631,7 @@ Lng32 HSGlobalsClass::processInternalSortNulls(Lng32 rowsRead, HSColGroupStruct
while (group)
{
- if (group->state != PENDING)
+ if (group->state != PENDING && group->state != OVERRAN)
{
group = group->next;
continue;
@@ -10541,48 +10701,84 @@ Lng32 HSGlobalsClass::processInternalSortNulls(Lng32 rowsRead, HSColGroupStruct
case REC_BYTE_V_ASCII:
case REC_BYTE_V_DOUBLE:
- // Set up elements of data array, which are pointers to varchar
- // values (2-byte length field followed by string). The length
- // we advance the ptr by includes an extra byte for alignment if
- // the varchar length is odd.
- // We also compute average varchar data size here which is not
- // something processInternalSortNulls() method should be doing.
- // This new code (computing avg size) will be moved to a separate
- // new method in the future.
- vcLen = (group->ISlength + VARCHAR_LEN_FIELD_IN_BYTES + (group->ISlength % 2));
- vchPtr = (ISVarChar*)group->nextData;
- dataPtr = (char*)group->strNextData;
- nullInd = group->nullIndics;
- sumSize = 0;
- for (i=0; i<rowsRead; i++)
- {
- vchPtr->setContent(dataPtr);
- // varchar type AND max long limit not reached.
- if (group->computeAvgVarCharSize() AND (!maxLongLimit))
+ {
+ // Set up elements of data array, which are pointers to varchar
+ // values (2-byte length field followed by string). The length
+ // we advance the ptr by includes an extra byte for alignment if
+ // the varchar length is odd.
+ // We also compute average varchar data size here which is not
+ // something processInternalSortNulls() method should be doing.
+ // This new code (computing avg size) will be moved to a separate
+ // new method in the future.
+ vcInflatedLen = group->inflatedVarcharContentSize();
+ vchPtr = (ISVarChar*)group->nextData;
+ dataPtr = (char*)group->strNextData;
+ inflatedDataPtr = (char*)group->varcharFetchBuffer;
+ nullInd = group->nullIndics;
+ NABoolean compacted = group->isCompacted();
+ Int64 nulls = 0; // Number of nulls in this batch of values
+ for (i=0; i<rowsRead; i++)
{
- // Not a NULL value.
- if (!nullInd OR *nullInd != -1)
- {
- sumSize += vchPtr->getLength();
- // if sumSize >= max limit for long, then stop and
- // calculate avg for the data we have so far.
- if (sumSize >= INT_MAX)
+ if (nullInd && *nullInd == -1)
{
- maxLongLimit = TRUE;
- group->avgVarCharSize = (double)sumSize / (double)(i+1);
+ nulls++;
+ if (compacted)
+ inflatedDataPtr += vcInflatedLen;
+ else
+ dataPtr += vcInflatedLen;
}
+ else
+ {
+ if (compacted)
+ {
+ vcCompactLen = HSColGroupStruct::varcharContentSize(*(Int16*)inflatedDataPtr);
+ group->sumSize += vcCompactLen;
+ if (group->state == PENDING) // that is, not OVERRAN
+ {
+
+ if (dataPtr + vcCompactLen > (char *)group->strData + group->strMemAllocated)
+ {
+ // We underestimated the space needed for compacted varchars,
+ // so don't save anymore. We'll continue to compute the actual
+ // average varchar length though.
+ group->state = OVERRAN;
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "Exhausted varchar compaction memory for column %s", group->colNames->data());
+ LM->Log(LM->msg);
+ }
+ }
+ else
+ {
+ memcpy(dataPtr, inflatedDataPtr, (ULng32)vcCompactLen);
+ inflatedDataPtr += vcInflatedLen;
+ vchPtr->setContent(dataPtr);
+ dataPtr += vcCompactLen;
+ }
+ }
+ }
+ else
+ {
+ vchPtr->setContent(dataPtr);
+ dataPtr += vcInflatedLen;
+ group->sumSize += vchPtr->getLength();
+ }
+
+ vchPtr++;
}
- } // varchar type
-
- if (nullInd) nullInd++;
- dataPtr += vcLen;
- vchPtr++;
- }
- if ( group->computeAvgVarCharSize() AND (group->avgVarCharSize == -1) )
- group->avgVarCharSize = (double)sumSize / (double)rowsRead;
- group->strNextData = dataPtr; // not affected by null processing
- processNullsForColumn(group, rowsRead, (ISVarChar*)NULL);
+ if (nullInd)
+ nullInd++;
+ }
+
+ group->nullCount += nulls;
+ group->rowsRead += rowsRead;
+ // compute average varchar size from running rows count and running sum of varchar sizes
+ if (group->rowsRead > 0)
+ group->avgVarCharSize = (double)group->sumSize / (double)group->rowsRead;
+ group->nextData = vchPtr;
+ group->strNextData = dataPtr;
+ }
break;
// LCOV_EXCL_START :rfi
@@ -10956,6 +11152,28 @@ Int32 HSGlobalsClass::selectSortBatch(Int64 rows,
// into account the second time.
while (group != NULL)
{
+ if (group->state == OVERRAN)
+ {
+ group->state = UNPROCESSED;
+
+ group->freeISMemory(TRUE,TRUE); // free old memory
+
+ // recalculate group->memNeeded based on what we now know about
+ // average varchar size
+
+ group->oldAvgVarCharSize = group->avgVarCharSize;
+ group->avgVarCharSize = -1; // to force a new computation
+ group->setISlength(group->ISlength,maxCharColumnLengthInBytes);
+
+ Int64 rows;
+ if (sampleRowCount > 0)
+ rows = sampleRowCount;
+ else
+ rows = actualRowCount;
+
+ getMemoryRequirementsForOneGroup(group,rows);
+ }
+
if (group->state == UNPROCESSED &&
group->memNeeded > 0 && // was set to 0 if exceeds address space
group->memNeeded < memLeft &&
@@ -11328,6 +11546,9 @@ Lng32 HSGlobalsClass::prepareToReadColumnsIntoMem(HSCursor *cursor, Int64 rows)
{
if (group->state == PENDING)
{
+ group->rowsRead = 0;
+ group->sumSize = 0;
+
if (firstExpn)
firstExpn = false;
else
@@ -11428,12 +11649,25 @@ Lng32 HSGlobalsClass::readColumnsIntoMem(HSCursor *cursor, Int64 rows)
retcode = cursor->setRowsetPointers(singleGroup,rowsetSize);
}
}
+
+ // Deallocate buffer uncompacted varchars were read into prior to being compacted.
+ HSColGroupStruct* group = singleGroup;
+ while (group != NULL)
+ {
+ if (group->state == PENDING && group->isCompacted())
+ {
+ NADELETEBASIC((short*)(group->varcharFetchBuffer), STMTHEAP);
+ group->varcharFetchBuffer = NULL;
+ }
+ group = group->next;
+ }
+
if (retcode < 0) HSHandleError(retcode) else retcode=0; // Set to 0 for warnings.
// some post-reading to memory processing to support MC in-memory computation
if ( performISForMC() )
{
- HSColGroupStruct* group = singleGroup;
+ group = singleGroup;
do
{
if (group->state == PENDING)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/ustat/hs_globals.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.h b/core/sql/ustat/hs_globals.h
index b9e50aa..65b88ca 100644
--- a/core/sql/ustat/hs_globals.h
+++ b/core/sql/ustat/hs_globals.h
@@ -1042,6 +1042,10 @@ enum SortState
{
UNPROCESSED, // Hasn't been selected yet
PENDING, // Selected for batch currently being processed
+ OVERRAN, // Selected for batch currently being processed but
+ // there isn't enough memory (happens only with
+ // varchar compaction where we underestimated average
+ // varchar size)
PROCESSED, // Already processed
DONT_TRY, // Memory allocation failed, don't try this one again
SKIP, // SKIP for the time being
@@ -1110,7 +1114,10 @@ struct HSColGroupStruct : public NABasicObject
HSColGroupStruct *prev; /* reverse list for SHOWSTATS */
HSColGroupStruct *mcis_next; /* For MC IS to point to next neighbor*/
char readTime[TIMESTAMP_CHAR_LEN+1]; /* read time; carry over to new hist */
- double coeffOfVar; /* coefficient of variation (skew of this hist) */
+ double coeffOfVar; /* coefficient of variation (skew of this hist) */
+ double oldAvgVarCharSize; /* average varchar size from previous histograms */
+ Int64 rowsRead; /* number of rows read for IS so far */
+ Int64 sumSize; /* sum of varchar size for IS so far */
double avgVarCharSize; /* average varchar size, -1 for other types */
char reason; /* automation reason */
char newReason; /* automation reason for updated hist */
@@ -1121,15 +1128,21 @@ struct HSColGroupStruct : public NABasicObject
SortState state; /* Internal sort status */
NABoolean delayedRead;
size_t memNeeded; /* memory required, in bytes */
+ size_t strMemAllocated; /* memory allocated, in bytes, for char data;
+ if compacted, this is just the area used
+ for compacted data */
void *data; /* Storage for column values */
void *nextData; /* Ptr to next place to store a value */
void *strData; /* Storage for char cols; data/nextdata */
void *strNextData; /* will be ptrs to this */
NABoolean strDataConsecutive; /* True if strData is as originally read */
+ void *varcharFetchBuffer; /* Direct fetch addr for varchar values that will be compacted */
short *nullIndics; /* Storage for null indicators */
Int64 nullCount; /* Number of null values */
+ NABoolean eligibleForVarCharCompaction; /* true if OK to use compaction on internal sort */
Lng32 ISdatatype; /* converted type for sorting */
Lng32 ISlength; /* len of converted type */
+ Lng32 ISvcLenUsed; /* varchar only; if compacted, is avg length which is usually < ISlength */
Lng32 ISprecision; /* prec of converted type */
Lng32 ISscale; /* scale of converted type */
NAString ISSelectExpn; /* select list expn to retrieve col */
@@ -1171,7 +1184,37 @@ struct HSColGroupStruct : public NABasicObject
NABoolean allocFilter(Lng32 count);
#endif
- inline NABoolean computeAvgVarCharSize() const
+ // @ZX Should we allow this to be called for non-varchar?
+ NABoolean isCompacted()
+ {
+ if (!DFS2REC::isAnyVarChar(ISdatatype))
+ return FALSE;
+ // TODO: next line causes a compilation error... why?
+ //HS_ASSERT(ISvcLenUsed > 0 && ISvcLenUsed <= ISlength);
+ return ISlength != ISvcLenUsed;
+ }
+
+ void setISlength(Lng32 len, Lng32 maxVarCharLengthInBytes);
+
+ // Size in bytes allocated for per varchar value in strData.
+ size_t varcharContentSize()
+ {
+ return varcharContentSize(ISvcLenUsed);
+ }
+
+ // For a compacted varchar, size in bytes of a single value in fetch buffer
+ // (prior to compaction).
+ size_t inflatedVarcharContentSize()
+ {
+ return varcharContentSize(ISlength);
+ }
+
+ // Calculate size to allocate for strData.
+ size_t strDataMemNeeded(Int64 rows);
+
+ // Calculate tha average actual varchar size for the stats
+ // collected on the current run.
+ NABoolean computeAvgVarCharSize() const
{
if ( (colCount == 1) AND
(DFS2REC::isAnyVarChar(colSet[0].datatype)) )
@@ -1188,6 +1231,19 @@ struct HSColGroupStruct : public NABasicObject
NABoolean recalcMemNeeded = FALSE);
void freeISMemory(NABoolean freeStrData = TRUE, NABoolean freeMCData=TRUE);
NAString generateTextForColumnCast();
+
+ // Returned value is the number of bytes needed to represent a single varchar
+ // value of the given length. The len parameter could be the declared length
+ // of a varchar column, or if varchars are being compacted, the estimated
+ // average actual length, or the actual length of a specific compacted varchar.
+ // To this we add the size of the length field, and a byte if necessary for the
+ // proper alignment of the Int16 length field.
+ static inline size_t varcharContentSize(Lng32 len)
+ {
+ return len // declared or avg estimated varchar len
+ + (len % 2) // possible alignment byte
+ + VARCHAR_LEN_FIELD_IN_BYTES; // size of len field
+ }
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3366fdba/core/sql/ustat/hs_read.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_read.cpp b/core/sql/ustat/hs_read.cpp
index b0f4ffd..07ffe74 100644
--- a/core/sql/ustat/hs_read.cpp
+++ b/core/sql/ustat/hs_read.cpp
@@ -2748,9 +2748,11 @@ void getPreviousUECRatios(HSColGroupStruct *groupList)
{
totalRows = 0;
totalUec = 0;
+ oldAvgVarcharSize = -1;
}
Int64 totalRows;
Int64 totalUec;
+ Int64 oldAvgVarcharSize;
};
// Allocate an array big enough to hold info on every column in the table.
@@ -2767,6 +2769,11 @@ void getPreviousUECRatios(HSColGroupStruct *groupList)
{
uecInfo[cursor.tableColNum_].totalRows = (Int64)cursor.totalRowCount_;
uecInfo[cursor.tableColNum_].totalUec = (Int64)cursor.totalUec_;
+ // the avgVarCharCount_ (V2 column in SB_HISTOGRAMS) is 100 times the
+ // average varchar length
+ if (cursor.avgVarCharCount_ > 0) // if the column is set
+ uecInfo[cursor.tableColNum_].oldAvgVarcharSize =
+ (Int64)(cursor.avgVarCharCount_+99)/100;
}
}
@@ -2778,10 +2785,11 @@ void getPreviousUECRatios(HSColGroupStruct *groupList)
colNum = group->colSet[0].colnum;
group->prevRowCount = uecInfo[colNum].totalRows;
group->prevUEC = uecInfo[colNum].totalUec;
+ group->oldAvgVarCharSize = uecInfo[colNum].oldAvgVarcharSize;
if (LM->LogNeeded())
{
- sprintf(LM->msg, "Existing histogram for column %s: rows = " PF64 ", UEC = " PF64,
- group->colSet[0].colname->data(), group->prevRowCount, group->prevUEC);
+ sprintf(LM->msg, "Existing histogram for column %s: rows = " PF64 ", UEC = " PF64 ", avgVarCharSize = %f",
+ group->colSet[0].colname->data(), group->prevRowCount, group->prevUEC, group->oldAvgVarCharSize);
LM->Log(LM->msg);
}
group = group->next;
[2/2] incubator-trafodion git commit: Merge [TRAFODION-2376] PR 1029
UPDATE STATS perf improvement for varchars
Posted by db...@apache.org.
Merge [TRAFODION-2376] PR 1029 UPDATE STATS perf improvement for varchars
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/29307b4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/29307b4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/29307b4c
Branch: refs/heads/master
Commit: 29307b4c0a888d203e47577e9c2bc125e6d2fa88
Parents: 8691d78 3366fdb
Author: Dave Birdsall <db...@apache.org>
Authored: Fri Mar 31 18:49:43 2017 +0000
Committer: Dave Birdsall <db...@apache.org>
Committed: Fri Mar 31 18:49:43 2017 +0000
----------------------------------------------------------------------
core/sqf/sql/scripts/analyzeULOG.py | 177 ++++++++++++++
core/sql/common/wstr.cpp | 41 ++++
core/sql/common/wstr.h | 11 +
core/sql/sqlcomp/DefaultConstants.h | 1 +
core/sql/sqlcomp/nadefaults.cpp | 1 +
core/sql/ustat/hs_cli.cpp | 7 +-
core/sql/ustat/hs_globals.cpp | 408 ++++++++++++++++++++++++-------
core/sql/ustat/hs_globals.h | 60 ++++-
core/sql/ustat/hs_read.cpp | 12 +-
9 files changed, 626 insertions(+), 92 deletions(-)
----------------------------------------------------------------------