You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by su...@apache.org on 2016/08/10 22:59:42 UTC
[1/6] incubator-trafodion git commit: [TRAFODION-2140] Move IUS
feature to open source
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 3cdea3f9b -> dda167cb0
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_globals.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.h b/core/sql/ustat/hs_globals.h
index e3d22e1..10381a8 100644
--- a/core/sql/ustat/hs_globals.h
+++ b/core/sql/ustat/hs_globals.h
@@ -73,7 +73,7 @@ class AbstractFastStatsHist;
Lng32 AddNecessaryColumns();
Lng32 AddAllColumnsForIUS();
-Lng32 createSampleOption(Lng32 sampleType, double samplePercent, NAString &sampleOpt,
+void createSampleOption(Lng32 sampleType, double samplePercent, NAString &sampleOpt,
Int64 sampleValue1=0, Int64 sampleValue2=0);
Lng32 doubleToHSDataBuffer(const double dbl, HSDataBuffer& dbf);
Lng32 managePersistentSamples();
@@ -1135,6 +1135,7 @@ struct HSColGroupStruct : public NABasicObject
Int64 prevUEC; /* uec from existing histogram */
Int64 colSecs; /* Time to sort/group data for column */
CountingBloomFilter* cbf; /* A bloom filter for IUS */
+ NAString& cbfFileNameSuffix() { return *colSet[0].colname; }
void* boundaryValues; /* List of bounary values for IUS */
void* MFVValues; /* List of MFV values for IUS */
@@ -1157,6 +1158,7 @@ struct HSColGroupStruct : public NABasicObject
/* are used by its neighbors. Used to compute group weight */
NABoolean allKeysInsertedIntoCBF;
+ Int32 backwardWarningCount; // for UERR_UNEXPECTED_BACKWARDS_DATA warnings
#ifdef _TEST_ALLOC_FAILURE
// Stuff used to test memory allocation failures.
@@ -1202,13 +1204,7 @@ class IUSValueIterator
virtual ~IUSValueIterator()
{}
- void init(HSColGroupStruct* group)
- {
- // Strings must be contiguous in the strData buffer for this iterator to
- // work correctly.
- HS_ASSERT(group->strDataConsecutive);
- vp = (T*)group->data;
- }
+ void init(HSColGroupStruct* group);
void next()
{
@@ -1357,6 +1353,9 @@ public:
void getMemoryRequirementsForOneMCGroup(HSColGroupStruct* group, Int64 rows);
static Int32 allocateMemoryForColumns(HSColGroupStruct* group, Int64 rows, HSColGroupStruct* mgr = NULL /* used for MC IS */);
+ static Int32 allocateMemoryForIUSColumns(HSColGroupStruct* group, Int64 rows,
+ HSColGroupStruct* delGroup, Int64 delRows,
+ HSColGroupStruct* insGroup, Int64 insRows);
// For internal sort or IUS, remove and count nulls for each column from the
// rowset just read.
@@ -1484,6 +1483,9 @@ public:
//Log the current contents of this class.
void log(HSLogMan* LM);
+ // Takes action necessary before throwing exception for an assertion failure.
+ void preAssertionFailure(const char* condition, const char* fileName, Lng32 lineNum);
+
// Derive a return code from the contents of the diagnostics area.
Lng32 getRetcodeFromDiags();
@@ -1502,14 +1504,20 @@ public:
NABoolean forceToFetch = TRUE);
Lng32 updatePersistentSampleTableForIUS(NAString& sampleTableName, double sampleRate,
NAString& targetTableName);
+ void generateIUSDeleteQuery(const NAString& smplTable, NAString& queryText);
+ void generateIUSSelectInsertQuery(const NAString& smplTable,
+ const NAString& sourceTable,
+ NAString& queryText);
+ void getCBFFilePrefix(NAString& sampleTableName, NAString& filePrefix);
void detectPersistentCBFsForIUS(NAString& sampleTableName, HSColGroupStruct *group);
+ Lng32 UpdateIUSPersistentSampleTable(Int64 oldSampleSize, Int64 requestedSampleSize, Int64& newSampleSize);
Lng32 readCBFsIntoMemForIUS(NAString& sampleTableName, HSColGroupStruct* group);
Lng32 writeCBFstoDiskForIUS(NAString& sampleTableName, HSColGroupStruct* group);
- Lng32 deletePersistentCBFsForIUS(NAString& sampleTableName, HSColGroupStruct* group);
+ Lng32 deletePersistentCBFsForIUS(NAString& sampleTableName, HSColGroupStruct* group, SortState stateToDelete);
void logDiagArea(const char* title);
- Lng32 begin_IUS_work(char* buffer);
+ Lng32 begin_IUS_work();
Lng32 end_IUS_work();
// Populate the hash table used to determine when a ustat statement has run
@@ -1579,6 +1587,7 @@ public:
Int64 numPartitions; /* # of partns in object */
NAString *hstogram_table; /* HISTOGRM table */
NAString *hsintval_table; /* HISTINTS table */
+ NAString *hsperssamp_table; /* PERSISTENT_SAMPLES table */
NAString *hssample_table; /* SAMPLING table */
NABoolean externalSampleTable; /* ownership of sample tab */
hs_table_type tableType; /* GUARDIAN | ANSI format */
@@ -1727,6 +1736,10 @@ private:
NABoolean internalSortWhenBetter,
NABoolean trySampleTableBypass = FALSE);
+ // After an allocation failure, this is called to reduce the amount of
+ // memory we estimate is available.
+ static void memReduceAllowance();
+
// When a memory allocation fails, return any memory already allocated for
// the group for internal sort, and set any PENDING columns back to
// UNPROCESSED state. This function cannot fail.
@@ -1756,6 +1769,17 @@ private:
// Collect statistics by incrementally updating persistent sample table and
// possibly histograms as well.
+ Lng32 doIUS(NABoolean& done);
+
+ // Collect stats by incrementally updating histograms where possible. Persistent
+ // sample is also incrementally updated.
+ Lng32 doFullIUS(Int64 currentSampleSize, Int64 futureSampleSize, NABoolean& done);
+
+ // Causes persistent sample table to be incrementally updated, and other
+ // preparatory tasks so RUS can be performed using persistent sample.
+ Lng32 prepareToUsePersistentSample (Int64 currentSampleSize, Int64 futureSampleSize);
+
+ // Incrementally update histograms for a selected batch of columns
Lng32 CollectStatisticsForIUS(Int64 currentSampleSize, Int64 futureSampleSize);
//
@@ -1798,6 +1822,18 @@ private:
HSInMemoryTable* iusSampleDeletedInMem;
HSInMemoryTable* iusSampleInsertedInMem;
+ // used by IUS code for clean up purposes
+ NABoolean sampleIExists_;
+
+ // For IUS, once the persistent sample table has been successfully updated
+ // in accordance with the IUS predicate, these ptrs will point to the requested
+ // (expected) and actual number of rows in the sample table. end_IUS_work will
+ // pass these ptrs to the function that updates the sample table's row in
+ // SB_PERSISTENT_SAMPLES. If non-null, the values are used for the corresponding
+ // columns in that table.
+ Int64* PST_IUSrequestedSampleRows_;
+ Int64* PST_IUSactualSampleRows_;
+
template <class T>
Int32 processIUSColumn(T* ptr,
const NAWchar* format,
@@ -1883,6 +1919,36 @@ private:
HSColGroupStruct* insGroup, Int64 insrows);
template <class T>
+ class HSHiLowValues
+ {
+ public:
+
+ NABoolean seenAtLeastOneValue_; // initially FALSE
+ // the next two are valid only if seenAtLeastOneValue_ is TRUE
+ T hiValue_; // highest value seen so far
+ T lowValue_; // lowest value seen so far
+
+ HSHiLowValues() : seenAtLeastOneValue_(FALSE) { };
+
+ void findHiLowValues(T& val)
+ {
+ if (seenAtLeastOneValue_)
+ {
+ if (val < lowValue_)
+ lowValue_ = val;
+ else if (val > hiValue_)
+ hiValue_ = val;
+ }
+ else
+ {
+ seenAtLeastOneValue_ = TRUE;
+ lowValue_ = val;
+ hiValue_ = val;
+ }
+ };
+ };
+
+ template <class T>
Int16 findInterval(Int16 numInt, T* boundaries, T& val)
{
Int16 low = 1;
@@ -1916,6 +1982,13 @@ private:
Int64 stmtStartTime;
NABoolean jitLogOn;
+ // For IUS, was the SB_PERSISTENT_SAMPLES row for the source table updated?
+ // The change is undone by the HSGlobalsClass dtor, so we need to account for
+ // the possibility that an IUS statement failed prior to making the change.
+ // Otherwise, a concurrent IUS operation could have its changes to the row
+ // overwritten.
+ NABoolean PSRowUpdated;
+
static THREAD_P NABoolean performISForMC_;
}; // class HSGlobalsClass
@@ -2165,6 +2238,8 @@ public:
void setHasNull(NABoolean val) { hasNull_ = val; }
void setIntBoundary(const Lng32 intNum, const char* value, Int16 len)
{ intArry_[intNum].boundary_.copyFrom(value, len, TRUE); }
+ void setIntBoundary(const Lng32 intNum, const HSDataBuffer & newBoundary)
+ { intArry_[intNum].boundary_ = newBoundary; }
void setIntMFVValue(const Lng32 intNum, const char* value, Int16 len)
{ intArry_[intNum].mostFreqVal_.copyFrom(value, len, TRUE); }
@@ -2449,11 +2524,6 @@ class HSInMemoryTable : public NABasicObject
void generateSelectDQuery(NAString& smplTable, NAString& queryTex);
void generateSelectIQuery(NAString& smplTable, NAString& queryText);
- void generateDeleteQuery(NAString& smplTable, NAString& queryText);
-
- void generateSelectInsertQuery(NAString& smplTable, NAString& sourceTable,
- NAString& queryText);
-
// method for algorithm 1
void generateDeleteQuery(NAString& smplTable, NAString& queryText, NABoolean rollback);
@@ -2462,6 +2532,16 @@ class HSInMemoryTable : public NABasicObject
Lng32 populate(NAString& queryText);
+ // The data is actually deallocated by calling freeISMemory() from
+ // HSGlobalsClass::incrementHistograms() for each column as soon as the
+ // column is successfully handled by IUS (the data is preserved for use
+ // by RUS/IS if IUS can't be performed). This function just resets the
+ // flag that would cause assertion failure when populate() is called, as
+ // it must be to load data for the next batch of IUS columns.
+ void depopulate() {
+ isPopulated_ = FALSE;
+ }
+
void logState(const char* title);
private:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_la.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_la.cpp b/core/sql/ustat/hs_la.cpp
index ea00289..068aef1 100644
--- a/core/sql/ustat/hs_la.cpp
+++ b/core/sql/ustat/hs_la.cpp
@@ -952,6 +952,18 @@ NABoolean HSHiveTableDef::objExists(NABoolean createExternalTable)
return TRUE;
}
+NAString HSHiveTableDef::getNodeName() const
+ {
+ HS_ASSERT(FALSE); // MP only
+ return "";
+ }
+
+NAString HSHiveTableDef::getCatalogLoc(formatType format) const
+ {
+ HS_ASSERT(FALSE); // MP only
+ return "";
+ }
+
NAString HSHiveTableDef::getHistLoc(formatType format) const
{
return HIVE_STATS_CATALOG "." HIVE_STATS_SCHEMA;
@@ -1020,6 +1032,18 @@ Lng32 HSHiveTableDef::DescribeColumnNames()
//=====================================================
//
+NAString HSHbaseTableDef::getNodeName() const
+ {
+ HS_ASSERT(FALSE); // MP only
+ return "";
+ }
+
+NAString HSHbaseTableDef::getCatalogLoc(formatType format) const
+ {
+ HS_ASSERT(FALSE); // MP only
+ return "";
+ }
+
NAString HSHbaseTableDef::getHistLoc(formatType format) const
{
if ( HSGlobalsClass::isNativeHbaseCat(getCatName(format))) {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_la.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_la.h b/core/sql/ustat/hs_la.h
index 3344feb..158b905 100644
--- a/core/sql/ustat/hs_la.h
+++ b/core/sql/ustat/hs_la.h
@@ -237,16 +237,8 @@ class HSHiveTableDef : public HSTableDef
{
return FALSE;
};
- NAString getNodeName() const
- {
- HS_ASSERT(FALSE); // MP only
- return "";
- }
- NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const
- {
- HS_ASSERT(FALSE); // MP only
- return "";
- }
+ NAString getNodeName() const;
+ NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const;
NAString getHistLoc(formatType format = INTERNAL_FORMAT) const;
Lng32 getFileType() const
{
@@ -348,16 +340,8 @@ class HSHbaseTableDef : public HSTableDef
{
return FALSE;
};
- NAString getNodeName() const
- {
- HS_ASSERT(FALSE); // MP only
- return "";
- }
- NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const
- {
- HS_ASSERT(FALSE); // MP only
- return "";
- }
+ NAString getNodeName() const;
+ NAString getCatalogLoc(formatType format = INTERNAL_FORMAT) const;
NAString getHistLoc(formatType format = INTERNAL_FORMAT) const;
Lng32 getFileType() const
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_log.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_log.h b/core/sql/ustat/hs_log.h
index da846dc..18e6ec5 100644
--- a/core/sql/ustat/hs_log.h
+++ b/core/sql/ustat/hs_log.h
@@ -135,21 +135,16 @@ private:
// Log the location of the error.
void HSFuncLogError(Lng32 error, char *filename, Lng32 lineno);
-// Wrapper to handle assertion failure.
+// Wrapper to handle assertion failure. Do not assert a condition with any
+// side effects, as it is evaluated a second time if false.
#define HS_ASSERT(b) \
+ { \
if (NOT (b)) \
{ \
- HSTranMan *TM = HSTranMan::Instance(); \
- HSLogMan *LM = HSLogMan::Instance(); \
- if (LM->LogNeeded()) \
- { \
- sprintf(LM->msg, "***[ERROR] INTERNAL ASSERTION (%s) AT %s:%i", "" # b "", __FILE__, __LINE__); \
- LM->Log(LM->msg); \
- } \
- if (TM->StartedTransaction()) \
- TM->Rollback(); \
+ GetHSContext()->preAssertionFailure("" # b "", __FILE__, __LINE__); \
CMPASSERT(b); \
- }
+ } \
+ }
//Ignore the following WARNINGS
// [6008] missing single-column histograms
@@ -157,11 +152,13 @@ void HSFuncLogError(Lng32 error, char *filename, Lng32 lineno);
// [4030] non-standard DATETIME format
// [4] internal Warning
#define HSFilterWarning(retcode) \
+ { \
if ((retcode == 6008) || \
(retcode == 6007) || \
(retcode == 4030) || \
(retcode == HS_WARNING)) \
- retcode = 0;
+ retcode = 0; \
+ }
// Map any error (<0) code other than HS_PKEY_FLOAT_ERROR to -1.
#define HSFilterError(retcode) \
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_parser.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_parser.cpp b/core/sql/ustat/hs_parser.cpp
index 11adbc1..d96acc7 100644
--- a/core/sql/ustat/hs_parser.cpp
+++ b/core/sql/ustat/hs_parser.cpp
@@ -462,15 +462,19 @@ Lng32 AddTableName( const hs_table_type type
*hs_globals->hsintval_table = getHistogramsTableLocation(hs_globals->catSch->data(), FALSE);
+ *hs_globals->hsperssamp_table = getHistogramsTableLocation(hs_globals->catSch->data(), FALSE);
+
NABoolean isHbaseOrHive = HSGlobalsClass::isHbaseCat(catName) ||
HSGlobalsClass::isHiveCat(catName);
if (isHbaseOrHive) {
hs_globals->hstogram_table->append(".").append(HBASE_HIST_NAME);
hs_globals->hsintval_table->append(".").append(HBASE_HISTINT_NAME);
+ hs_globals->hsperssamp_table->append(".").append(HBASE_PERS_SAMP_NAME);
} else {
hs_globals->hstogram_table->append(".HISTOGRAMS");
hs_globals->hsintval_table->append(".HISTOGRAM_INTERVALS");
+ hs_globals->hsperssamp_table->append(".PERSISTENT_SAMPLES");
}
}
else
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_update.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_update.cpp b/core/sql/ustat/hs_update.cpp
index e8ca28e..4dd70d6 100644
--- a/core/sql/ustat/hs_update.cpp
+++ b/core/sql/ustat/hs_update.cpp
@@ -501,9 +501,6 @@ Lng32 UpdateStats(char *input, NABoolean requestedByCompiler)
HSClearCLIDiagnostics();
- if ( hs_globals_obj.canDoIUS() )
- hs_globals_obj.end_IUS_work();
-
hs_globals_y = NULL;
// Remove IUS persistent sample if necessary.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_yacc.y
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_yacc.y b/core/sql/ustat/hs_yacc.y
index 0007659..0ee1ca4 100644
--- a/core/sql/ustat/hs_yacc.y
+++ b/core/sql/ustat/hs_yacc.y
@@ -353,19 +353,18 @@ histogram_options : CLEAR
{
hs_globals_y->optFlags |= CLEAR_OPT;
}
- | CREATE SAMPLE sample_clause_for_createremove
+ | CREATE SAMPLE random_clause
{
- if (hs_globals_y->optFlags & SAMPLE_ALL)
+ if (hs_globals_y->optFlags & SAMPLE_RAND_2)
{
- HSFuncMergeDiags(- UERR_INVALID_OPTION,
- "ALL",
- "RANDOM X PERCENT or r ROWS");
+ HSFuncMergeDiags(-UERR_IUS_WRONG_RANDOM);
hs_globals_y->parserError = HSGlobalsClass::ERROR_SEMANTICS;
return -1;
}
+
hs_globals_y->optFlags |= CREATE_SAMPLE_OPT;
}
- | REMOVE SAMPLE sample_clause_for_createremove
+ | REMOVE SAMPLE
{
hs_globals_y->optFlags |= REMOVE_SAMPLE_OPT;
}
@@ -402,37 +401,6 @@ histogram_options : CLEAR
| /* empty */
;
-sample_clause_for_createremove: ALL
- {
- hs_globals_y->optFlags |= SAMPLE_ALL;
- }
- | int_number ROWS
- {
- if ($1 <= 0)
- {
- HSFuncMergeDiags(- UERR_INVALID_OPTION,
- "SAMPLE ROWS",
- "an integer greater than or equal to 0 and within limits");
- hs_globals_y->parserError = HSGlobalsClass::ERROR_SEMANTICS;
- return -1;
- }
- hs_globals_y->optFlags |= SAMPLE_BASIC_1;
- hs_globals_y->sampleValue1 = $1;
- }
- | RANDOM real_number TOK_PERCENT
- {
- if ($2 <= 0 || $2 >= 100)
- {
- HSFuncMergeDiags(- UERR_INVALID_OPTION,
- "PERCENT",
- "a value between 0 and 100");
- hs_globals_y->parserError = HSGlobalsClass::ERROR_SEMANTICS;
- return -1;
- }
- hs_globals_y->optFlags |= SAMPLE_RAND_1;
- hs_globals_y->sampleValue1 = (Int64)($2 * HS_SAMP_PCNT_UPSCALE);
- }
-;
on_clause : ON predefined_groups
| ON predefined_groups ',' regular_group_list
@@ -463,7 +431,10 @@ incremental_clause : INCREMENTAL WHERE WHERE_CONDITION
if (LM->LogNeeded() )
LM->Log("incremental clause identified");
-
+ if (CmpCommon::getDefault(USTAT_INCREMENTAL_UPDATE_STATISTICS) == DF_OFF) {
+ HSFuncMergeDiags(-UERR_IUS_IS_DISABLED);
+ }
+
if (CmpCommon::getDefault(USTAT_IUS_SIMPLE_SYNTAX) == DF_ON) {
// Via grammar, the incremental clause can only used without the
// on_clause. So it is safe to set the IUS_OPT flag.
@@ -585,17 +556,17 @@ sample_clause_body : SAMPLE
if (LM->LogNeeded() )
LM->Log("Creation of persistent sample table for IUS requested");
- if (hs_globals_y->optFlags & (REG_GROUP_OPT | EVERYCOL_OPT | EVERYKEY_OPT))
- HSFuncMergeDiags(-UERR_WRONG_ON_CLAUSE_FOR_IUS, "PERSISTENT");
- else if (hs_globals_y->optFlags & SAMPLE_RAND_2)
+ // Formerly there was code here to limit PERSISTENT to
+ // ON EXISTING COLUMNS and ON NECESSARY COLUMNS (EXISTING_OPT and
+ // NECESSARY_OPT in the optFlags). But there doesn't seem to be
+ // a compelling reason for this limitation. The persistent sample
+ // table will have all the columns of the base table regardless
+ // of the ON clause.
+
+ if (hs_globals_y->optFlags & SAMPLE_RAND_2)
HSFuncMergeDiags(-UERR_IUS_WRONG_RANDOM);
else
- {
- // This assert is here to make sure we covered all other possible ON
- // clauses in the check above.
- HS_ASSERT(hs_globals_y->optFlags & (EXISTING_OPT | NECESSARY_OPT));
- hs_globals_y->optFlags |= IUS_PERSIST;
- }
+ hs_globals_y->optFlags |= IUS_PERSIST;
}
| SAMPLE random_clause rowcount_clause
| SAMPLE periodic_clause
[4/6] incubator-trafodion git commit: [TRAFODION-2140] Move IUS
feature to open source
Posted by su...@apache.org.
[TRAFODION-2140] Move IUS feature to open source
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/021ebd6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/021ebd6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/021ebd6c
Branch: refs/heads/master
Commit: 021ebd6cfec18db6f616fbfc08ee2a4bf3bb4b33
Parents: 37901fb
Author: Barry Fritchman <bl...@gmail.com>
Authored: Tue Aug 9 17:19:07 2016 +0000
Committer: Barry Fritchman <bl...@gmail.com>
Committed: Tue Aug 9 17:19:07 2016 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 10 +-
core/sql/common/ComSmallDefs.h | 4 +
core/sql/common/NAString.cpp | 7 +
core/sql/optimizer/RelSample.cpp | 1 -
core/sql/regress/compGeneral/EXPECTED023 | 559 ++++++
core/sql/regress/compGeneral/FILTER023 | 36 +
core/sql/regress/compGeneral/TEST023 | 273 +++
core/sql/regress/core/EXPECTED116 | 3 +
core/sql/regress/executor/EXPECTED130 | 1 +
core/sql/regress/hive/EXPECTED009 | 1 +
core/sql/regress/privs1/EXPECTED132 | 5 +
core/sql/regress/privs1/EXPECTED133 | Bin 26390 -> 27095 bytes
core/sql/regress/privs1/EXPECTED141 | Bin 113222 -> 123489 bytes
core/sql/regress/privs1/TEST132 | 2 +-
core/sql/regress/privs2/EXPECTED138 | 24 +-
core/sql/regress/privs2/EXPECTED140 | 25 +-
core/sql/regress/seabase/EXPECTED002 | 50 +-
core/sql/regress/seabase/EXPECTED012 | 1 +
core/sql/regress/seabase/EXPECTED026 | 24 +-
core/sql/regress/tools/runregr_compGeneral.ksh | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 5 +-
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 69 +-
core/sql/sqlcomp/CmpSeabaseDDLmd.h | 25 +
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 8 +-
core/sql/ustat/hs_auto.h | 4 +-
core/sql/ustat/hs_cli.cpp | 831 +++++----
core/sql/ustat/hs_cli.h | 81 +-
core/sql/ustat/hs_const.h | 11 +-
core/sql/ustat/hs_globals.cpp | 1822 +++++++++++++------
core/sql/ustat/hs_globals.h | 110 +-
core/sql/ustat/hs_la.cpp | 24 +
core/sql/ustat/hs_la.h | 24 +-
core/sql/ustat/hs_log.h | 21 +-
core/sql/ustat/hs_parser.cpp | 4 +
core/sql/ustat/hs_update.cpp | 3 -
core/sql/ustat/hs_yacc.y | 65 +-
36 files changed, 2987 insertions(+), 1148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt
index f21cf09..d39769b 100644
--- a/core/sql/bin/SqlciErrors.txt
+++ b/core/sql/bin/SqlciErrors.txt
@@ -1870,7 +1870,8 @@ drop the default context
9225 ZZZZZ 99999 BEGINNER MINOR LOGONLY Incremental UPDATE STATISTICS: the total unique count for column $0~String0 changed more than the percentage specified by CQD USTAT_IUS_TOTAL_UEC_CHANGE_THRESHOLD. A regular UPDATE STATISTICS is performed instead.
9226 ZZZZZ 99999 BEGINNER MINOR LOGONLY Incremental UPDATE STATISTICS: one interval for column $0~String0 is reset to zero. A regular UPDATE STATISTICS is performed instead.
9227 ZZZZZ 99999 BEGINNER MINOR LOGONLY Incremental UPDATE STATISTICS: an interval for column $0~String0 is reduced to zero rows using IUS. A regular UPDATE STATISTICS is performed instead.
-9230 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS: cannot allocate enough memory. A regular UPDATE STATISTICS is performed instead.
+9228 ZZZZZ 99999 BEGINNER MINOR LOGONLY There were no sample tables to drop.
+9230 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS: $0~int0 columns could not be processed due to insufficient memory, and are being processed by regular UPDATE STATISTICS instead.
9231 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS: only RANDOM <percentage> PERCENT is allowed as the sampling specification with PERSISTENT.
9232 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS: cannot proceed because of the on-going IUS transaction originated at $0~String0.
9233 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS: could not store search condition used for IUS because it exceeds $0~int0 characters.
@@ -1885,10 +1886,13 @@ drop the default context
9242 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Unable to create schema for Hive statistics.
9243 ZZZZZ 99999 BEGINNER MAJOR DBADMIN This UPDATE STATISTICS command may take too long. It is recommended to use the SAMPLE clause instead. If you wish to do this without a SAMPLE clause, specify NO SAMPLE explicitly.
9244 ZZZZZ 99999 BEGINNER MAJOR DBADMIN UPDATE STATISTICS is not allowed in a user transaction.
-9245 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU ---- unused ----
+9245 ZZZZZ 99999 BEGINNER INFRM DBADMIN Unexpected out-of-order data encountered during histogram construction on column $0~string0; this might result in later 6004 warnings.
9246 ZZZZZ 99999 BEGINNER MAJOR DBADMIN UPDATE STATISTICS is not supported on LOB columns. Column $0~String0 is a LOB column.
9247 ZZZZZ 99999 BEGINNER MAJOR DBADMIN UPDATE STATISTICS is not supported on volatile tables presently.
-9250 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Last UPDATE STATISTICS error.
+9248 ZZZZZ 99999 BEGINNER MAJOR DBADMIN UPDATE STATISTICS failed due to a memory allocation failure.
+9249 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Incremental UPDATE STATISTICS is disabled.
+9250 ZZZZZ 99999 BEGINNER MINOR LOGONLY Incremental UPDATE STATISTICS: non-NULL values added to previously all-NULL histogram for column $0~string0. A regular UPDATE STATISTICS is performed instead.
+9259 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Last UPDATE STATISTICS error.
10000 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU Sort Error: First Sort error
10001 ZZZZZ 99999 ADVANCED MAJOR DIALOUT Sort Error : No error text defined. Unexpected error. $0~String0
10002 ZZZZZ 99999 UUUUUUUU UUUUU UUUUUUU ---- unused ----
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/common/ComSmallDefs.h
----------------------------------------------------------------------
diff --git a/core/sql/common/ComSmallDefs.h b/core/sql/common/ComSmallDefs.h
index 5a25b62..68de7b7 100644
--- a/core/sql/common/ComSmallDefs.h
+++ b/core/sql/common/ComSmallDefs.h
@@ -104,18 +104,22 @@ typedef NABoolean ComBoolean;
#define HIVE_SYSTEM_SCHEMA "HIVE"
#define HIVE_STATS_CATALOG "TRAFODION"
#define HIVE_STATS_SCHEMA "\"_HIVESTATS_\""
+#define HIVE_STATS_SCHEMA_NO_QUOTES "_HIVESTATS_"
#define HIVE_EXT_SCHEMA_PREFIX "_HV_"
#define HBASE_SYSTEM_CATALOG "HBASE"
#define HBASE_SYSTEM_SCHEMA "HBASE"
#define HBASE_HIST_NAME "SB_HISTOGRAMS"
#define HBASE_HISTINT_NAME "SB_HISTOGRAM_INTERVALS"
+#define HBASE_PERS_SAMP_NAME "SB_PERSISTENT_SAMPLES"
#define HBASE_HIST_PK "SB_HISTOGRAMS_PK"
#define HBASE_HISTINT_PK "SB_HISTOGRAM_INTERVALS_PK"
+#define HBASE_PERS_SAMP_PK "SB_PERSISTENT_SAMPLES_PK"
#define HBASE_EXT_SCHEMA_PREFIX "_HB_"
#define HBASE_STATS_CATALOG "TRAFODION"
#define HBASE_STATS_SCHEMA "\"_HBASESTATS_\""
+#define HBASE_STATS_SCHEMA_NO_QUOTES "_HBASESTATS_"
// default null format for data in hive files.
#define HIVE_DEFAULT_NULL_STRING "\\N"
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/common/NAString.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/NAString.cpp b/core/sql/common/NAString.cpp
index 3b9830e..0816baa 100644
--- a/core/sql/common/NAString.cpp
+++ b/core/sql/common/NAString.cpp
@@ -280,6 +280,13 @@ NAString Int64ToNAString(Int64 l)
return NAString(resultstr);
}
+NAString RealToNAString(double d)
+{
+ char resultstr[200];
+ sprintf(resultstr,"%G",d);
+ return NAString(resultstr);
+}
+
NAString &replaceAll(NAString &source, const NAString &searchFor,
const NAString &replaceWith)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/optimizer/RelSample.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelSample.cpp b/core/sql/optimizer/RelSample.cpp
index 56d88b8..92f318a 100644
--- a/core/sql/optimizer/RelSample.cpp
+++ b/core/sql/optimizer/RelSample.cpp
@@ -925,7 +925,6 @@ RelExpr *RelSample::bindNode(BindWA *bindWA)
RelExpr* myChild = child(0);
if (myChild->getOperatorType() == REL_SCAN &&
(static_cast<Scan*>(myChild))->isHbaseTable() &&
- myChild->selectionPred().entries() == 0 &&
isSimpleRandomRelative() &&
trafSampleRate <= 1.0f)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/compGeneral/EXPECTED023
----------------------------------------------------------------------
diff --git a/core/sql/regress/compGeneral/EXPECTED023 b/core/sql/regress/compGeneral/EXPECTED023
new file mode 100644
index 0000000..5fea598
--- /dev/null
+++ b/core/sql/regress/compGeneral/EXPECTED023
@@ -0,0 +1,559 @@
+>>-- create the database used for the tests
+>>
+>>create schema compgeneral_test023;
+
+--- SQL operation complete.
+>>
+>>set schema compgeneral_test023;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+
+--- SQL operation complete.
+>>
+>>CREATE TABLE stest
++>(
++>c1 integer not null,
++>c2 integer not null,
++>c3 integer not null,
++>PRIMARY KEY (C1 ASC)
++>)
++>SALT USING 4 PARTITIONS
++>ON (C1)
++>;
+
+--- SQL operation complete.
+>>
+>>
+>>upsert using load into stest
++>select
++> x1+10*x2+100*x3+1000*x4+10000*x5,
++> x2,
++> x3
++>-- the from clause below creates 100,000 rows, the cross product of
++>-- 5 copies of { 0, ... 9 }
++> from (values(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)) T(x1)
++>transpose 0,1,2,3,4,5,6,7,8,9 as x2
++>transpose 0,1,2,3,4,5,6,7,8,9 as x3
++>transpose 0,1,2,3,4,5,6,7,8,9 as x4
++>transpose 0,1,2,3,4,5,6,7,8,9 as x5;
+
+--- 100000 row(s) inserted.
+>>
+>>
+>>CREATE TABLE stestc
++>(
++>c1 char(12) not null,
++>c2 char(12) not null,
++>c3 char(12) not null,
++>PRIMARY KEY (C1 ASC)
++>)
++>SALT USING 4 PARTITIONS
++>ON (C1)
++>;
+
+--- SQL operation complete.
+>>
+>>upsert using load into stestc
++>select
++> x1 || x2 || x3 || x4 || x5,
++> x2 || x4 || x1,
++> x5 || x3
++>-- the from clause below creates 100,000 rows, the cross product of
++>-- 5 copies of { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' }
++> from (values('a'),('b'),('c'),('d'),('e'),('f'),('g'),('h'),('i'),('j')) T(x1)
++>transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x2
++>transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x3
++>transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x4
++>transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x5;
+
+--- 100000 row(s) inserted.
+>>
+>>
+>>create table stest_empty like stest with partitions;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+
+--- SQL operation complete.
+>> -- should be just stest, stest_empty, stestc; no sb_* tables yet
+>>
+>>?section ustat1p
+>>
+>>-- positive tests
+>>
+>>-- create a persistent sample
+>>update statistics for table stest create sample random 20 percent;
+
+--- SQL operation complete.
+>>
+>>prepare s1 from
++>select object_name,sample_name,reason,last_where_predicate
++> from "_MD_".objects o join sb_persistent_samples s
++> on o.object_uid = s.table_uid
++> where object_name = ? and object_type = 'BT';
+
+--- SQL command prepared.
+>>
+>>execute s1 using 'STEST';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STEST TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_6_1456279355_95736 M
+
+--- 1 row(s) selected.
+>>-- should see one row
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_6_1456279355_95736
+
+--- SQL operation complete.
+>> -- should be stest, stest_empty, stestc, sb_persistent_samples + a sample table
+>>
+>>-- create another one, showing its replacement
+>>update statistics for table stest create sample random 10 percent;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_5_1456279391_83165
+
+--- SQL operation complete.
+>> -- should be stest, stest_empty, stestc, sb_persistent_samples + a different sample table
+>>
+>>execute s1 using 'STEST';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STEST TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_5_1456279391_83165 M
+
+--- 1 row(s) selected.
+>>-- should see one row
+>>
+>>-- remove a persistent sample table
+>>update statistics for table stest remove sample;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+
+--- SQL operation complete.
+>> -- should be stest, stest_empty, stestc, sb_persistent_samples only
+>>
+>>execute s1 using 'STEST';
+
+--- 0 row(s) selected.
+>>-- should see zero rows
+>>
+>>-- create a persistent sample table using PERSISTENT
+>>
+>>update statistics for table stest on every column sample random 10 percent persistent;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_72_1456279415_388472
+
+--- SQL operation complete.
+>> -- should be stest, stest_empty, stestc, sb_persistent_samples, sb_hist* + another sample table
+>>
+>>execute s1 using 'STEST';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STEST TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_72_1456279415_388472 I
+
+--- 1 row(s) selected.
+>>-- should see one row
+>>
+>>-- do it again showing its replacement
+>>update statistics for table stest on every column sample random 10 percent persistent;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be stest, stest_empty, stestc, sb_persistent_samples, sb_hist* + another sample table
+>>
+>>execute s1 using 'STEST';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STEST TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_88_1456279440_120688 I
+
+--- 1 row(s) selected.
+>>-- should see one row
+>>
+>>-- change the data in the table and do an INCREMENTAL update stats
+>>insert into stest values (100000,1,1),
++> (100001,1,1),
++> (100002,1,1),
++> (100003,1,1),
++> (100004,1,1),
++> (100005,1,1),
++> (100006,1,1),
++> (100007,1,1),
++> (100008,1,1),
++> (100009,1,1),
++> (100010,2,1),
++> (100011,2,1),
++> (100012,2,1),
++> (100013,2,1),
++> (100014,2,1),
++> (100015,2,1),
++> (100016,2,1),
++> (100017,2,1),
++> (100018,2,1),
++> (100019,2,1),
++> (100020,3,1),
++> (100021,3,1),
++> (100022,3,1),
++> (100023,3,1),
++> (100024,3,1),
++> (100025,3,1),
++> (100026,3,1),
++> (100027,3,1),
++> (100028,3,1),
++> (100029,3,1);
+
+--- 30 row(s) inserted.
+>>
+>>-- These two CQDs are to get around annoying 9222 and 9224 warnings
+>>-- that happen non-deterministically due to "gaps" in the sample histograms.
+>>cqd USTAT_IUS_INTERVAL_ROWCOUNT_CHANGE_THRESHOLD '0.15';
+
+--- SQL operation complete.
+>>cqd USTAT_IUS_INTERVAL_UEC_CHANGE_THRESHOLD '0.15';
+
+--- SQL operation complete.
+>>
+>>update statistics for table stest on existing columns incremental where c1 >= 100000;
+
+--- SQL operation complete.
+>>
+>>execute s1 using 'STEST';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STEST TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_88_1456279440_120688 I c1 >= 100000
+
+--- 1 row(s) selected.
+>>
+>>-- similar tests, but using character data types
+>>update statistics for table stestc on every column sample random 10 percent persistent;
+
+--- SQL operation complete.
+>>
+>>insert into stestc values ('naaaa','abc','ba'),
++> ('naaab','abc','ba'),
++> ('naaac','abc','ba'),
++> ('naaad','abc','ba'),
++> ('naaae','abc','ba'),
++> ('naaaf','abc','ba'),
++> ('naaag','abc','ba'),
++> ('naaah','abc','ba'),
++> ('naaai','abc','ba'),
++> ('naaaj','abc','ba'),
++> ('naaba','bhe','ba'),
++> ('naabb','bhe','ba'),
++> ('naabc','bhe','ba'),
++> ('naabd','bhe','ba'),
++> ('naabe','bhe','ba'),
++> ('naabf','bhe','ba'),
++> ('naabg','bhe','ba'),
++> ('naabh','bhe','ba'),
++> ('naabi','bhe','ba'),
++> ('naabj','bhe','ba'),
++> ('naaca','fib','ba'),
++> ('naacb','fib','ba'),
++> ('naacc','fib','ba'),
++> ('naacd','fib','ba'),
++> ('naace','fib','ba'),
++> ('naacf','fib','ba'),
++> ('naacg','fib','ba'),
++> ('naach','fib','ba'),
++> ('naaci','fib','ba'),
++> ('naacj','fib','ba');
+
+--- 30 row(s) inserted.
+>>
+>>update statistics for table stestc on existing columns incremental where c1 >= 'naaaa';
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>>
+>>execute s1 using 'STESTC';
+
+OBJECT_NAME SAMPLE_NAME REASON LAST_WHERE_PREDICATE
+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ------ -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+
+STESTC TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPLE_85_1456279493_417385 I c1 >= 'naaaa'
+
+--- 1 row(s) selected.
+>>
+>>?section ustat1n
+>>-- negative tests
+>>
+>>-- attempt to create a sample on an empty table
+>>update statistics for table stest_empty create sample random 10 percent;
+
+*** ERROR[9207] The specified SAMPLE option generated an empty sample set. Modify the SAMPLE option and resubmit.
+
+--- SQL operation failed with errors.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be the same as previous "get tables"
+>>
+>>execute s1 using 'STEST_EMPTY';
+
+--- 0 row(s) selected.
+>>-- should be zero rows
+>>
+>>update statistics for table stest_empty on every column sample random 10 percent persistent;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be the same as previous "get tables"
+>>
+>>execute s1 using 'STEST_EMPTY';
+
+--- 0 row(s) selected.
+>>-- should be zero rows
+>>
+>>-- attempt to remove a non-existant persistent sample
+>>update statistics for table stest_empty remove sample;
+
+*** WARNING[9228] There were no sample tables to drop.
+
+--- SQL operation completed with warnings.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be the same as previous "get tables"
+>>
+>>-- attempt to do incremental with an invalid where clause
+>>update statistics for table stest on existing columns incremental where 1;
+
+*** ERROR[9200] UPDATE STATISTICS for table TRAFODION.COMPGENERAL_TEST023.STEST encountered an error (15001) from statement HSCursor::prepareRowsetInternal().
+
+*** ERROR[15001] A syntax error occurred at or before:
+SELECT "C1", "C2", "C3", "_SALT_" FROM TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPL
+E_88_1456279440_120688 WHERE 1 FOR READ UNCOMMITTED ACCESS;
+ ^ (120 characters from start of SQL statement)
+
+*** ERROR[8822] The statement was not prepared.
+
+*** ERROR[9200] UPDATE STATISTICS for table TRAFODION.COMPGENERAL_TEST023.STEST encountered an error (15001) from statement POPULATE_FROM_QUERY.
+
+*** ERROR[15001] A syntax error occurred at or before:
+SELECT "C1", "C2", "C3", "_SALT_" FROM TRAFODION.COMPGENERAL_TEST023.TRAF_SAMPL
+E_88_1456279440_120688 WHERE 1 FOR READ UNCOMMITTED ACCESS;
+ ^ (120 characters from start of SQL statement)
+
+*** ERROR[8822] The statement was not prepared.
+
+--- SQL operation failed with errors.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be the same as previous "get tables"
+>>
+>>-- attempt to do incremental when no persistent sample exists
+>>update statistics for table stest_empty on existing columns incremental where c1 >= 100000;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+STEST
+STESTC
+STEST_EMPTY
+TRAF_SAMPLE_85_1456279493_417385
+TRAF_SAMPLE_88_1456279440_120688
+
+--- SQL operation complete.
+>> -- should be the same as previous "get tables"
+>>
+>>
+>>?section clnup
+>>
+>>drop table stest;
+
+--- SQL operation complete.
+>>drop table stestc;
+
+--- SQL operation complete.
+>>drop table stest_empty;
+
+--- SQL operation complete.
+>>
+>>get tables;
+
+Tables in Schema TRAFODION.COMPGENERAL_TEST023
+==============================================
+
+SB_HISTOGRAMS
+SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
+
+--- SQL operation complete.
+>> -- sample tables should be gone too
+>>
+>>drop schema compgeneral_test023 cascade;
+
+--- SQL operation complete.
+>> -- cascade needed because of sb_* tables
+>>
+>>log;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/compGeneral/FILTER023
----------------------------------------------------------------------
diff --git a/core/sql/regress/compGeneral/FILTER023 b/core/sql/regress/compGeneral/FILTER023
new file mode 100755
index 0000000..dc7f367
--- /dev/null
+++ b/core/sql/regress/compGeneral/FILTER023
@@ -0,0 +1,36 @@
+#! /bin/sh
+# @@@ START COPYRIGHT @@@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# @@@ END COPYRIGHT @@@
+
+# The names of sample tables are generated from non-deterministic
+# criteria; we filter that out.
+
+fil=$1
+if [ "$fil" = "" ]; then
+ echo "Usage: $0 filename"
+ exit 1
+fi
+
+sed "
+s/TRAF_SAMPLE_[0-9]*_[0-9]*_[0-9]*/TRAF_SAMPLE_nn_nnnnnnnnnn_nnnnnn/g
+s/E_[0-9]*_[0-9]*_[0-9]* WHERE/E_nn_nnnnnnnnnn_nnnnnn WHERE/g
+" $fil
+
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/compGeneral/TEST023
----------------------------------------------------------------------
diff --git a/core/sql/regress/compGeneral/TEST023 b/core/sql/regress/compGeneral/TEST023
new file mode 100755
index 0000000..01ab49b
--- /dev/null
+++ b/core/sql/regress/compGeneral/TEST023
@@ -0,0 +1,273 @@
+-- Test: TEST023 (CompGeneral)
+-- Functionality: Tests of the incremental UPDATE STATISTICS feature.
+--
+-- @@@ START COPYRIGHT @@@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+--
+-- @@@ END COPYRIGHT @@@
+
+?section clean_up
+
+set schema compgeneral_test023;
+
+drop schema compgeneral_test023 cascade;
+
+?section create_db
+
+log LOG023 clear;
+-- create the database used for the tests
+
+create schema compgeneral_test023;
+
+set schema compgeneral_test023;
+
+get tables;
+
+CREATE TABLE stest
+(
+c1 integer not null,
+c2 integer not null,
+c3 integer not null,
+PRIMARY KEY (C1 ASC)
+)
+SALT USING 4 PARTITIONS
+ON (C1)
+;
+
+
+upsert using load into stest
+select
+ x1+10*x2+100*x3+1000*x4+10000*x5,
+ x2,
+ x3
+-- the from clause below creates 100,000 rows, the cross product of
+-- 5 copies of { 0, ... 9 }
+ from (values(0),(1),(2),(3),(4),(5),(6),(7),(8),(9)) T(x1)
+transpose 0,1,2,3,4,5,6,7,8,9 as x2
+transpose 0,1,2,3,4,5,6,7,8,9 as x3
+transpose 0,1,2,3,4,5,6,7,8,9 as x4
+transpose 0,1,2,3,4,5,6,7,8,9 as x5;
+;
+
+CREATE TABLE stestc
+(
+c1 char(12) not null,
+c2 char(12) not null,
+c3 char(12) not null,
+PRIMARY KEY (C1 ASC)
+)
+SALT USING 4 PARTITIONS
+ON (C1)
+;
+
+upsert using load into stestc
+select
+ x1 || x2 || x3 || x4 || x5,
+ x2 || x4 || x1,
+ x5 || x3
+-- the from clause below creates 100,000 rows, the cross product of
+-- 5 copies of { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' }
+ from (values('a'),('b'),('c'),('d'),('e'),('f'),('g'),('h'),('i'),('j')) T(x1)
+transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x2
+transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x3
+transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x4
+transpose 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j' as x5;
+;
+
+create table stest_empty like stest with partitions;
+
+get tables; -- should be just stest, stest_empty, stestc; no sb_* tables yet
+
+?section ustat1p
+
+-- positive tests
+
+-- create a persistent sample
+update statistics for table stest create sample random 20 percent;
+
+prepare s1 from
+select object_name,sample_name,reason,last_where_predicate
+ from "_MD_".objects o join sb_persistent_samples s
+ on o.object_uid = s.table_uid
+ where object_name = ? and object_type = 'BT';
+
+execute s1 using 'STEST';
+-- should see one row
+
+get tables; -- should be stest, stest_empty, stestc, sb_persistent_samples + a sample table
+
+-- create another one, showing its replacement
+update statistics for table stest create sample random 10 percent;
+
+get tables; -- should be stest, stest_empty, stestc, sb_persistent_samples + a different sample table
+
+execute s1 using 'STEST';
+-- should see one row
+
+-- remove a persistent sample table
+update statistics for table stest remove sample;
+
+get tables; -- should be stest, stest_empty, stestc, sb_persistent_samples only
+
+execute s1 using 'STEST';
+-- should see zero rows
+
+-- create a persistent sample table using PERSISTENT
+
+update statistics for table stest on every column sample random 10 percent persistent;
+
+get tables; -- should be stest, stest_empty, stestc, sb_persistent_samples, sb_hist* + another sample table
+
+execute s1 using 'STEST';
+-- should see one row
+
+-- do it again showing its replacement
+update statistics for table stest on every column sample random 10 percent persistent;
+
+get tables; -- should be stest, stest_empty, stestc, sb_persistent_samples, sb_hist* + another sample table
+
+execute s1 using 'STEST';
+-- should see one row
+
+-- change the data in the table and do an INCREMENTAL update stats
+insert into stest values (100000,1,1),
+ (100001,1,1),
+ (100002,1,1),
+ (100003,1,1),
+ (100004,1,1),
+ (100005,1,1),
+ (100006,1,1),
+ (100007,1,1),
+ (100008,1,1),
+ (100009,1,1),
+ (100010,2,1),
+ (100011,2,1),
+ (100012,2,1),
+ (100013,2,1),
+ (100014,2,1),
+ (100015,2,1),
+ (100016,2,1),
+ (100017,2,1),
+ (100018,2,1),
+ (100019,2,1),
+ (100020,3,1),
+ (100021,3,1),
+ (100022,3,1),
+ (100023,3,1),
+ (100024,3,1),
+ (100025,3,1),
+ (100026,3,1),
+ (100027,3,1),
+ (100028,3,1),
+ (100029,3,1);
+
+-- These two CQDs are to get around annoying 9222 and 9224 warnings
+-- that happen non-deterministically due to "gaps" in the sample histograms.
+cqd USTAT_IUS_INTERVAL_ROWCOUNT_CHANGE_THRESHOLD '0.15';
+cqd USTAT_IUS_INTERVAL_UEC_CHANGE_THRESHOLD '0.15';
+
+update statistics for table stest on existing columns incremental where c1 >= 100000;
+
+execute s1 using 'STEST';
+
+-- similar tests, but using character data types
+update statistics for table stestc on every column sample random 10 percent persistent;
+
+insert into stestc values ('naaaa','abc','ba'),
+ ('naaab','abc','ba'),
+ ('naaac','abc','ba'),
+ ('naaad','abc','ba'),
+ ('naaae','abc','ba'),
+ ('naaaf','abc','ba'),
+ ('naaag','abc','ba'),
+ ('naaah','abc','ba'),
+ ('naaai','abc','ba'),
+ ('naaaj','abc','ba'),
+ ('naaba','bhe','ba'),
+ ('naabb','bhe','ba'),
+ ('naabc','bhe','ba'),
+ ('naabd','bhe','ba'),
+ ('naabe','bhe','ba'),
+ ('naabf','bhe','ba'),
+ ('naabg','bhe','ba'),
+ ('naabh','bhe','ba'),
+ ('naabi','bhe','ba'),
+ ('naabj','bhe','ba'),
+ ('naaca','fib','ba'),
+ ('naacb','fib','ba'),
+ ('naacc','fib','ba'),
+ ('naacd','fib','ba'),
+ ('naace','fib','ba'),
+ ('naacf','fib','ba'),
+ ('naacg','fib','ba'),
+ ('naach','fib','ba'),
+ ('naaci','fib','ba'),
+ ('naacj','fib','ba');
+
+update statistics for table stestc on existing columns incremental where c1 >= 'naaaa';
+
+get tables;
+
+execute s1 using 'STESTC';
+
+?section ustat1n
+-- negative tests
+
+-- attempt to create a sample on an empty table
+update statistics for table stest_empty create sample random 10 percent;
+
+get tables; -- should be the same as previous "get tables"
+
+execute s1 using 'STEST_EMPTY';
+-- should be zero rows
+
+update statistics for table stest_empty on every column sample random 10 percent persistent;
+
+get tables; -- should be the same as previous "get tables"
+
+execute s1 using 'STEST_EMPTY';
+-- should be zero rows
+
+-- attempt to remove a non-existant persistent sample
+update statistics for table stest_empty remove sample;
+
+get tables; -- should be the same as previous "get tables"
+
+-- attempt to do incremental with an invalid where clause
+update statistics for table stest on existing columns incremental where 1;
+
+get tables; -- should be the same as previous "get tables"
+
+-- attempt to do incremental when no persistent sample exists
+update statistics for table stest_empty on existing columns incremental where c1 >= 100000;
+
+get tables; -- should be the same as previous "get tables"
+
+
+?section clnup
+
+drop table stest;
+drop table stestc;
+drop table stest_empty;
+
+get tables; -- sample tables should be gone too
+
+drop schema compgeneral_test023 cascade; -- cascade needed because of sb_* tables
+
+log;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/core/EXPECTED116
----------------------------------------------------------------------
diff --git a/core/sql/regress/core/EXPECTED116 b/core/sql/regress/core/EXPECTED116
index 3932ba7..0441135 100644
--- a/core/sql/regress/core/EXPECTED116
+++ b/core/sql/regress/core/EXPECTED116
@@ -645,6 +645,7 @@ Tables in Schema TRAFODION.T116SCH1
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>create table t116t2 (a int);
@@ -657,6 +658,7 @@ Tables in Schema TRAFODION.T116SCH1
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T116T2
--- SQL operation complete.
@@ -685,6 +687,7 @@ Tables in Schema TRAFODION.T116SCH1
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T116T1
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/executor/EXPECTED130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/EXPECTED130 b/core/sql/regress/executor/EXPECTED130
index af65951..bb19a2b 100644
--- a/core/sql/regress/executor/EXPECTED130
+++ b/core/sql/regress/executor/EXPECTED130
@@ -1360,6 +1360,7 @@ LOBDescHandle__05102114855562636318_0001
LOBMD__05102114855562636318
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
TLOB130TS2
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/hive/EXPECTED009
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED009 b/core/sql/regress/hive/EXPECTED009
index 7d27a24..372d5de 100644
--- a/core/sql/regress/hive/EXPECTED009
+++ b/core/sql/regress/hive/EXPECTED009
@@ -264,6 +264,7 @@ Tables in Schema TRAFODION.HIVE_T009
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T009T1
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs1/EXPECTED132
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs1/EXPECTED132 b/core/sql/regress/privs1/EXPECTED132
index 0e9828c..d84aee8 100644
--- a/core/sql/regress/privs1/EXPECTED132
+++ b/core/sql/regress/privs1/EXPECTED132
@@ -222,6 +222,7 @@ Tables in Schema TRAFODION.T132SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -263,6 +264,7 @@ Tables in Schema TRAFODION.T132SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T132T1
T132T2
T132T3
@@ -623,6 +625,7 @@ Tables in Schema TRAFODION.T132SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -1868,6 +1871,7 @@ Tables in Schema TRAFODION.T132SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -1891,6 +1895,7 @@ Tables in Schema TRAFODION.T132SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T132T1
T132T2
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs1/EXPECTED133
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs1/EXPECTED133 b/core/sql/regress/privs1/EXPECTED133
index d9fb353..a6d6307 100644
Binary files a/core/sql/regress/privs1/EXPECTED133 and b/core/sql/regress/privs1/EXPECTED133 differ
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs1/EXPECTED141
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs1/EXPECTED141 b/core/sql/regress/privs1/EXPECTED141
index 49eca21..308aae9 100644
Binary files a/core/sql/regress/privs1/EXPECTED141 and b/core/sql/regress/privs1/EXPECTED141 differ
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs1/TEST132
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs1/TEST132 b/core/sql/regress/privs1/TEST132
index 41f56f9..30e846f 100755
--- a/core/sql/regress/privs1/TEST132
+++ b/core/sql/regress/privs1/TEST132
@@ -51,7 +51,7 @@ exit;
?section clean_up
set schema t132sch;
drop sequence t132_team_number_sequence;
-drop schema t132sch cascade;
+cleanup schema t132sch;
?section set_up
create shared schema t132sch;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs2/EXPECTED138
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs2/EXPECTED138 b/core/sql/regress/privs2/EXPECTED138
index 92caae5..ece9b94 100644
--- a/core/sql/regress/privs2/EXPECTED138
+++ b/core/sql/regress/privs2/EXPECTED138
@@ -19,6 +19,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -34,6 +35,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>create table user1_t1 (c1 int not null primary key, c2 int);
@@ -118,6 +120,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>create table user1_t1 (c1 int not null primary key, c2 int);
@@ -198,6 +201,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -386,6 +390,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -407,6 +412,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -424,6 +430,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -444,6 +451,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -461,6 +469,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -472,6 +481,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -493,6 +503,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -513,6 +524,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>create table user1_t1 (c1 int not null primary key, c2 int);
@@ -597,6 +609,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -790,6 +803,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -999,6 +1013,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -1201,6 +1216,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -1218,6 +1234,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -1272,6 +1289,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>create table user1_t1 (c1 int not null primary key, c2 int);
@@ -1511,6 +1529,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
USER1_T1
USER1_T2
@@ -1528,6 +1547,7 @@ Tables in Schema TRAFODION.T138SCH
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
--- SQL operation complete.
>>
@@ -1739,7 +1759,7 @@ End of MXCI Session
(EXPR)
--------------------
- 11
+ 13
--- 1 row(s) selected.
>>
@@ -1753,7 +1773,7 @@ End of MXCI Session
(EXPR)
--------------------
- 11
+ 13
--- 1 row(s) selected.
>>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/privs2/EXPECTED140
----------------------------------------------------------------------
diff --git a/core/sql/regress/privs2/EXPECTED140 b/core/sql/regress/privs2/EXPECTED140
index 84543e2..b31683f 100644
--- a/core/sql/regress/privs2/EXPECTED140
+++ b/core/sql/regress/privs2/EXPECTED140
@@ -206,6 +206,7 @@ GAMES
PLAYERS
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
TEAMS
--- SQL operation complete.
@@ -318,6 +319,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES VI SQL_USER1 SQL_USER2 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -327,10 +329,11 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER1 SQL_USER2 SI----- SI-----
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
---- 16 row(s) selected.
+--- 18 row(s) selected.
>>
>>exit;
@@ -486,6 +489,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES VI SQL_USER1 SQL_USER2 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -496,11 +500,12 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER2 SQL_USER3 SI----- -I-----
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER2 SQL_USER3 S-D---- S-D----
---- 19 row(s) selected.
+--- 21 row(s) selected.
>>
>>exit;
@@ -630,6 +635,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.PLAYERS_ON_TEAM VI -2 SQL_USER3 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -641,12 +647,13 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER3 SQL_USER4 -I----- NONE
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER2 SQL_USER3 S-D---- S-D----
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER3 SQL_USER4 S------ NONE
---- 22 row(s) selected.
+--- 24 row(s) selected.
>>
>>exit;
@@ -736,6 +743,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.PLAYERS_ON_TEAM VI -2 SQL_USER3 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -748,12 +756,13 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER3 SQL_USER4 -I----- NONE
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER2 SQL_USER3 S-D---- S-D----
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER3 SQL_USER4 S------ NONE
---- 23 row(s) selected.
+--- 25 row(s) selected.
>>
>>exit;
@@ -867,6 +876,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.PLAYERS_ON_TEAM VI -2 SQL_USER3 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -886,12 +896,13 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER3 SQL_USER4 -I----- NONE
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER2 SQL_USER3 S-D---- S-D----
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER3 SQL_USER4 S------ NONE
---- 30 row(s) selected.
+--- 32 row(s) selected.
>>
>>-- user6 tries to grant select to user7
>>grant select on games to sql_user7 by sql_user6;
@@ -943,6 +954,7 @@ TRAFODION.T140_SHARED_VIEWS.HOME_TEAM_GAMES
TRAFODION.T140_SHARED_VIEWS.PLAYERS_ON_TEAM VI -2 SQL_USER3 S------ NONE
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAMS BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.SB_HISTOGRAM_INTERVALS BT -2 DB__ROOT SIDU-R- SIDU-R-
+TRAFODION.T140_SHARED_VIEWS.SB_PERSISTENT_SAMPLES BT -2 DB__ROOT SIDU-R- SIDU-R-
TRAFODION.T140_SHARED_VIEWS.T140_L1 LB -2 DB__ROOT ---UG-- ---UG--
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR -2 DB__ROOT ------E ------E
TRAFODION.T140_SHARED_VIEWS.T140_TRANSLATEPRIVSBIT UR DB__ROOT -1 ------E NONE
@@ -960,11 +972,12 @@ TRAFODION.T140_USER1_PRIVATE.PLAYERS
TRAFODION.T140_USER1_PRIVATE.PLAYERS BT SQL_USER3 SQL_USER4 -I----- NONE
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.SB_HISTOGRAM_INTERVAL BT -2 SQL_USER1 SIDU-R- SIDU-R-
+TRAFODION.T140_USER1_PRIVATE.SB_PERSISTENT_SAMPLES BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT -2 SQL_USER1 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER1 SQL_USER2 SIDU-R- SIDU-R-
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER2 SQL_USER3 S-D---- S-D----
TRAFODION.T140_USER1_PRIVATE.TEAMS BT SQL_USER3 SQL_USER4 S------ NONE
---- 28 row(s) selected.
+--- 30 row(s) selected.
>>
>>log;
[6/6] incubator-trafodion git commit: Merge [TRAFODION-2140] PR-651
Add capability for incremental Update Statistics
Posted by su...@apache.org.
Merge [TRAFODION-2140] PR-651 Add capability for incremental Update Statistics
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/dda167cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/dda167cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/dda167cb
Branch: refs/heads/master
Commit: dda167cb0164866e9561b79ec2d5c524552800e9
Parents: 3cdea3f d7ebca7
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Aug 10 22:59:11 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Aug 10 22:59:11 2016 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 10 +-
core/sql/common/ComSmallDefs.h | 4 +
core/sql/common/NAString.cpp | 7 +
core/sql/optimizer/RelSample.cpp | 1 -
core/sql/regress/compGeneral/EXPECTED023 | 559 ++++++
core/sql/regress/compGeneral/FILTER023 | 36 +
core/sql/regress/compGeneral/TEST023 | 273 +++
core/sql/regress/core/EXPECTED116 | 3 +
core/sql/regress/executor/EXPECTED130 | 1 +
core/sql/regress/hive/EXPECTED009 | 1 +
core/sql/regress/privs1/EXPECTED132 | 5 +
core/sql/regress/privs1/EXPECTED133 | Bin 26390 -> 27095 bytes
core/sql/regress/privs1/EXPECTED141 | Bin 113222 -> 123489 bytes
core/sql/regress/privs1/TEST132 | 2 +-
core/sql/regress/privs2/EXPECTED138 | 24 +-
core/sql/regress/privs2/EXPECTED140 | 25 +-
core/sql/regress/seabase/EXPECTED002 | 61 +-
core/sql/regress/seabase/EXPECTED012 | 1 +
core/sql/regress/seabase/EXPECTED026 | 24 +-
core/sql/regress/tools/runregr_compGeneral.ksh | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp | 5 +-
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 69 +-
core/sql/sqlcomp/CmpSeabaseDDLmd.h | 25 +
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 8 +-
core/sql/ustat/hs_auto.h | 4 +-
core/sql/ustat/hs_cli.cpp | 831 +++++----
core/sql/ustat/hs_cli.h | 81 +-
core/sql/ustat/hs_const.h | 11 +-
core/sql/ustat/hs_globals.cpp | 1822 +++++++++++++------
core/sql/ustat/hs_globals.h | 110 +-
core/sql/ustat/hs_la.cpp | 24 +
core/sql/ustat/hs_la.h | 24 +-
core/sql/ustat/hs_log.h | 21 +-
core/sql/ustat/hs_parser.cpp | 4 +
core/sql/ustat/hs_update.cpp | 3 -
core/sql/ustat/hs_yacc.y | 65 +-
36 files changed, 2994 insertions(+), 1152 deletions(-)
----------------------------------------------------------------------
[5/6] incubator-trafodion git commit: [TRAFODION-2140] Add code for
Incremental Update Statistics
Posted by su...@apache.org.
[TRAFODION-2140] Add code for Incremental Update Statistics
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d7ebca78
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d7ebca78
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d7ebca78
Branch: refs/heads/master
Commit: d7ebca78ed7bb9f5a05c91912138acebf1548a00
Parents: 021ebd6 87e4ca4
Author: Barry Fritchman <bl...@gmail.com>
Authored: Wed Aug 10 08:31:19 2016 +0000
Committer: Barry Fritchman <bl...@gmail.com>
Committed: Wed Aug 10 08:31:19 2016 +0000
----------------------------------------------------------------------
.../jdbc/t4/TrafT4PreparedStatement.java | 2 +-
.../odbc/src/odbc/nsksrvrcore/srvrothers.cpp | 77 ++-
core/sqf/sql/scripts/keepHBaseUp.py | 113 ++++
.../transactional/TransactionManager.java | 520 ++++++++-----------
core/sqf/src/tm/tmlogging.cpp | 138 +++--
core/sqf/src/tm/tmlogging.h | 2 -
core/sql/bin/SqlciErrors.txt | 2 +
core/sql/comexe/ComTdbExeUtil.cpp | 3 +-
core/sql/comexe/ComTdbExeUtil.h | 89 +++-
core/sql/common/CharType.cpp | 4 +-
core/sql/common/OperTypeEnum.h | 1 +
core/sql/common/str.cpp | 23 +-
core/sql/executor/ExExeUtil.h | 48 +-
core/sql/executor/ExExeUtilCommon.cpp | 31 ++
core/sql/executor/ExExeUtilGet.cpp | 396 ++++++++++++--
core/sql/executor/ExExplain.cpp | 26 +-
core/sql/executor/ExHdfsScan.cpp | 52 +-
core/sql/executor/HBaseClient_JNI.cpp | 48 +-
core/sql/executor/HBaseClient_JNI.h | 2 +
core/sql/exp/ExpHbaseInterface.cpp | 20 +
core/sql/exp/ExpHbaseInterface.h | 3 +
core/sql/exp/ExpLOBaccess.cpp | 2 +-
core/sql/generator/GenRelExeUtil.cpp | 39 +-
core/sql/generator/GenRelScan.cpp | 13 +-
core/sql/optimizer/BindRelExpr.cpp | 131 ++++-
core/sql/optimizer/EncodedKeyValue.cpp | 18 +-
core/sql/optimizer/OptRange.cpp | 17 +-
core/sql/optimizer/RelExeUtil.cpp | 9 +-
core/sql/optimizer/RelExeUtil.h | 23 +-
core/sql/optimizer/RelExpr.cpp | 17 +
core/sql/optimizer/RelSet.h | 23 +
core/sql/parser/sqlparser.y | 40 +-
core/sql/regress/compGeneral/EXPECTED001.SB | 10 +
core/sql/regress/compGeneral/EXPECTED042 | 8 +-
core/sql/regress/compGeneral/TEST001 | 1 +
core/sql/regress/hive/EXPECTED002 | 4 +-
core/sql/regress/seabase/EXPECTED002 | 174 ++++++-
core/sql/regress/seabase/EXPECTED031 | 124 ++++-
core/sql/regress/seabase/FILTER002 | 4 +-
core/sql/regress/seabase/TEST002 | 38 +-
core/sql/regress/seabase/TEST031 | 19 +
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 2 +
core/sql/sqlcomp/nadefaults.cpp | 3 +-
.../java/org/trafodion/sql/HBaseClient.java | 124 ++++-
.../java/org/trafodion/sql/TrafRegionStats.java | 118 ++++-
.../trafodion/jdbc_test/TestGetIndexInfo.java | 196 +++++++
46 files changed, 2129 insertions(+), 628 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d7ebca78/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d7ebca78/core/sql/regress/seabase/EXPECTED002
----------------------------------------------------------------------
diff --cc core/sql/regress/seabase/EXPECTED002
index 2888c5b,73a9b91..2f3645c
--- a/core/sql/regress/seabase/EXPECTED002
+++ b/core/sql/regress/seabase/EXPECTED002
@@@ -447,36 -459,9 +461,38 @@@ Stats Summar
Stats Details
=============
+ RegionServer: ansharma-3:44229
RegionNum: 1
- RegionName: TRAFODION.T002SCH.SB_HISTOGRAM_INTERVALS/1470338660549
+ RegionName: TRAFODION.T002SCH.SB_HISTOGRAM_INTERVALS/72bc6242dfa139b77bd15125cf46593f
+ NumStores: 2
+ NumStoreFiles: 0
+ UncompressedSize: 0 (less than 1MB)
+ StoreFileSize: 0 (less than 1MB)
+ MemStoreSize: 0 (less than 1MB)
+ ReadRequestsCount: 0
+ WriteRequestsCount: 0
+
+
+Stats Summary
+=============
+
+ ObjectName: TRAFODION.T002SCH.SB_PERSISTENT_SAMPLES
+ NumRegions: 1
+ RegionsLocation: /hbase/data/default
+ TotalNumStores: 2
+ TotalNumStoreFiles: 0
+ TotalUncompressedSize: 0
+ TotalStoreFileSize: 0
+ TotalMemStoreSize: 0
+ TotalReadRequestsCount: 0
+ TotalWriteRequestsCount: 0
+
+Stats Details
+=============
+
++ RegionServer: edev06:54527
+ RegionNum: 1
+ RegionName: TRAFODION.T002SCH.SB_PERSISTENT_SAMPLES/121ea772487c23e7f165e29db4647fd4
NumStores: 2
NumStoreFiles: 0
UncompressedSize: 0 (less than 1MB)
@@@ -804,4 -782,97 +827,99 @@@ Stats Summar
--- SQL operation complete.
>>
+ >>-- where preds
+ >>select cast(trim(schema_name) || '.' || trim(object_name) as char(30) character set iso88591),
+ +> region_num, region_name, num_stores, num_store_files
+ +> from table(region stats(t002t1))
+ +> where object_name = 'T002T1';
+
+ (EXPR) REGION_NUM REGION_NAME NUM_STORES NUM_STORE_FILES
+ ------------------------------ -------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ----------- ---------------
+
+ T002SCH.T002T1 1 TRAFODION.T002SCH.T002T1/1470338662761 2 0
+
+ --- 1 row(s) selected.
+ >>
+ >>select cast(trim(object_name) as char(30) character set iso88591),
+ +> sum(store_file_size)
+ +> from table(region statistics (using (select * from (
+ +> get tables in schema t002sch, no header, return full names) x(a) )))
+ +> where object_name like 'T002T1%'
+ +> group by 1 order by 2 desc;
+
+ (EXPR) (EXPR)
+ ------------------------------ --------------------
+
+ T002T1 0
+
+ --- 1 row(s) selected.
+ >>
+ >>obey TEST002(cluster_stats);
+ >>invoke table(cluster stats());
+
+ -- Definition of Trafodion table TRAFODION.T002SCH.EXE_UTIL_CLUSTER_STATS__
+ -- Definition current Thu Aug 4 19:25:21 2016
+
+ (
+ REGION_SERVER CHAR(256 BYTES) CHARACTER SET UTF8 COLLATE
+ DEFAULT NO DEFAULT NOT NULL NOT DROPPABLE
+ , REGION_NAME CHAR(512 BYTES) CHARACTER SET UTF8 COLLATE
+ DEFAULT NO DEFAULT NOT NULL NOT DROPPABLE
+ , CATALOG_NAME CHAR(256 BYTES) CHARACTER SET UTF8 COLLATE
+ DEFAULT NO DEFAULT NOT NULL NOT DROPPABLE
+ , SCHEMA_NAME CHAR(256 BYTES) CHARACTER SET UTF8 COLLATE
+ DEFAULT NO DEFAULT NOT NULL NOT DROPPABLE
+ , OBJECT_NAME CHAR(256 BYTES) CHARACTER SET UTF8 COLLATE
+ DEFAULT NO DEFAULT NOT NULL NOT DROPPABLE
+ , NUM_STORES INT NO DEFAULT NOT NULL NOT DROPPABLE
+ , NUM_STORE_FILES INT NO DEFAULT NOT NULL NOT DROPPABLE
+ , STORE_FILE_UNCOMP_SIZE LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
+ , STORE_FILE_SIZE LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
+ , MEM_STORE_SIZE LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
+ , READ_REQUESTS_COUNT LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
+ , WRITE_REQUESTS_COUNT LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
+ )
+
+ --- SQL operation complete.
+ >>
+ >>select cast(trim(schema_name) || '.' || trim(object_name) as char(30) character set iso88591),
+ +> sum(store_file_size)
+ +> from table(cluster stats())
+ +> where schema_name = 'T002SCH'
+ +> group by 1 order by 2 desc;
+
+ (EXPR) (EXPR)
+ ------------------------------ --------------------
+
+ T002SCH.SB_HISTOGRAM_INTERVALS 0
+ T002SCH.T002T1I2 0
-T002SCH.SB_HISTOGRAMS 0
+ T002SCH.T002T1 0
-T002SCH.T002T2 0
++T002SCH.SB_PERSISTENT_SAMPLES 0
+ T002SCH.T002T1I1 0
++T002SCH.T002T2 0
++T002SCH.SB_HISTOGRAMS 0
+ T002SCH.T002T2I1 0
+
---- 7 row(s) selected.
++--- 8 row(s) selected.
+ >>
+ >>select cast('RegionName: ' || trim(region_name) as char(30) character set iso88591),
+ +> sum(store_file_size)
+ +> from table(cluster stats())
+ +> where schema_name = 'T002SCH'
+ +> group by 1 order by 2 desc;
+
+ (EXPR) (EXPR)
+ ------------------------------ --------------------
+
+ RegionName: 1470338670665 0
+ RegionName: 1470338698762 0
+ RegionName: 1470338658677 0
+ RegionName: 1470338660549 0
+ RegionName: 1470338665125 0
+ RegionName: 1470338701334 0
+ RegionName: 1470338662761 0
++RegionName: 1470816094440 0
+
---- 7 row(s) selected.
++--- 8 row(s) selected.
+ >>
>>log;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d7ebca78/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
[2/6] incubator-trafodion git commit: [TRAFODION-2140] Move IUS
feature to open source
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_globals.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_globals.cpp b/core/sql/ustat/hs_globals.cpp
index aa1c860..43f582e 100644
--- a/core/sql/ustat/hs_globals.cpp
+++ b/core/sql/ustat/hs_globals.cpp
@@ -120,6 +120,8 @@ Lng32 setBufferValue(MCWrapper& value, const HSColGroupStruct *mgroup, HSDataBuf
template <class T>
void createHistogram(HSColGroupStruct *group, Lng32 numIntervals, Int64 estRowCount, NABoolean usingSample, T* dummyPtr);
+static Lng32 drop_I(NAString& sampTblName);
+
//
// Initialize the GLOBAL instances of ISFixedChar and ISVarChar values.
// See the "as lightweight as possible" comments in hs_globals.h
@@ -1413,7 +1415,8 @@ HSColGroupStruct::HSColGroupStruct()
mcis_colsMissingMap(NULL), mcis_memFreed(FALSE),
mcis_totalMCmemNeeded(0), mcis_groupHead(TRUE), mcis_next(NULL), mcis_readAsIs (FALSE),
delayedRead(FALSE), cbf(NULL),
- boundaryValues(NULL), MFVValues(NULL), allKeysInsertedIntoCBF(FALSE)
+ boundaryValues(NULL), MFVValues(NULL), allKeysInsertedIntoCBF(FALSE),
+ backwardWarningCount(0)
{
strcpy(readTime, "0001-01-01 00:00:00"); // default if new
#ifdef _TEST_ALLOC_FAILURE
@@ -1583,6 +1586,13 @@ NABoolean HSColGroupStruct::allocateISMemory(Int64 rows,
// avoid freeing strData when this fn is called to remove the old data array.
void HSColGroupStruct::freeISMemory(NABoolean freeStrData, NABoolean freeMCData)
{
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "Freeing IS memory for column %s", colNames->data());
+ LM->Log(LM->msg);
+ }
+
// used by MC in-memory since a column might have been processed but kept
// in memory to be used by MCs
mcis_memFreed = TRUE;
@@ -2835,6 +2845,7 @@ HSGlobalsClass::HSGlobalsClass(ComDiagsArea &diags)
numPartitions(0),
hstogram_table(new(STMTHEAP) NAString(STMTHEAP)),
hsintval_table(new(STMTHEAP) NAString(STMTHEAP)),
+ hsperssamp_table(new(STMTHEAP) NAString(STMTHEAP)),
hssample_table(new(STMTHEAP) NAString(STMTHEAP)),
statstime(new(STMTHEAP) NAString(STMTHEAP)),
@@ -2865,9 +2876,13 @@ HSGlobalsClass::HSGlobalsClass(ComDiagsArea &diags)
//iusSampleInMem(NULL),
iusSampleDeletedInMem(NULL),
iusSampleInsertedInMem(NULL),
+ sampleIExists_(FALSE),
+ PST_IUSrequestedSampleRows_(NULL),
+ PST_IUSactualSampleRows_(NULL),
sampleRateAsPercetageForIUS(0),
minRowCtPerPartition_(-1),
sample_I_generated(FALSE),
+ PSRowUpdated(FALSE),
jitLogThreshold(0),
stmtStartTime(0),
jitLogOn(FALSE),
@@ -2898,6 +2913,16 @@ HSGlobalsClass::HSGlobalsClass(ComDiagsArea &diags)
HSGlobalsClass::~HSGlobalsClass()
{
+ // If this was an IUS execution, make sure the row for the source table in
+ // SB_PERSISTENT_SAMPLES is modified to reflect that an IUS operation is no
+ // longer in progress.
+ if (PSRowUpdated)
+ end_IUS_work();
+
+ // Used in end_IUS_work(), must call it first.
+ NADELETEBASIC(PST_IUSrequestedSampleRows_, STMTHEAP);
+ NADELETEBASIC(PST_IUSactualSampleRows_, STMTHEAP);
+
// reset the parser flags that were set in the constructor
SQL_EXEC_ResetParserFlagsForExSqlComp_Internal(savedParserFlags);
@@ -3035,7 +3060,7 @@ Lng32 HSGlobalsClass::Initialize()
inserts, deletes, updates,
numPartitions,
minRowCtPerPartition_,
- optFlags & SAMPLE_REQUESTED);
+ optFlags & (SAMPLE_REQUESTED | IUS_OPT));
LM->StopTimer();
if (LM->LogNeeded())
{
@@ -3403,7 +3428,7 @@ NABoolean HSGlobalsClass::isAuthorized(NABoolean isShowStats)
return TRUE;
// no privilege support available for hbase and hive tables
- assert (objDef->getNATable());
+ HS_ASSERT (objDef->getNATable());
if (CmpSeabaseDDL::isHbase(objDef->getCatName()) || isHiveCat(objDef->getCatName()))
return TRUE;
@@ -3621,14 +3646,104 @@ void HSGlobalsClass::startJitLogging(const char* checkPointName, Int64 elapsedSe
}
}
+// The optimal degree of parallelism for a LOAD or UPSERT is the number of
+// partitions of the source table. This forces that by setting the cqd
+// PARALLEL_NUM_ESPS. Note that when the default for AGGRESSIVE_ESP_ALLOCATION_PER_CORE
+// is permanently changed to 'ON', we may be able to remove this.
+// tblDef -- ptr to HSTableDef from which to get the catalog and schema name of
+// the source table.
+// tblName -- unqualified name of the source table. If NULL, then the source
+// table is the one represented by tblDef.
+// Returns TRUE if the cqd was successfully set, FALSE otherwise. If TRUE is returned,
+// then resetEspParallelism() may be called to reset the cqd.
+static NABoolean setEspParallelism(HSTableDef* tblDef, const char* tblName = NULL)
+{
+ HSLogMan *LM = HSLogMan::Instance();
+ Lng32 retcode = 0;
+ Lng32 numPartitions = 0;
+ if (!tblName)
+ numPartitions = tblDef->getNumPartitions();
+ else
+ {
+ HSCursor cursor;
+ NAString numPartitionsQuery;
+ numPartitionsQuery.append("select t.num_salt_partns from \"_MD_\".OBJECTS O, \"_MD_\".TABLES T where o.catalog_name = '")
+ .append(tblDef->getCatName())
+ .append("' and o.schema_name = '")
+ .append(tblDef->getSchemaName())
+ .append("' and o.object_name = '")
+ .append(tblName)
+ .append("' and o.object_uid = t.table_uid");
+ retcode = cursor.fetchNumColumn(numPartitionsQuery, &numPartitions, NULL);
+ if (retcode != 0)
+ {
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "PARALLEL_NUM_ESPS not set; query to get # partitions received sqlcode %d", retcode);
+ LM->Log(LM->msg);
+ }
+ return FALSE;
+ }
+ }
+ NABoolean espCQDUsed = FALSE;
+ if (numPartitions > 1)
+ {
+ char temp[25];
+ sprintf(temp, "'%d'", numPartitions);
+ NAString espsCQD = "CONTROL QUERY DEFAULT PARALLEL_NUM_ESPS ";
+ espsCQD += temp;
+ retcode = HSFuncExecQuery(espsCQD);
+ if (retcode < 0)
+ {
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "cqd statement to set PARALLEL_NUM_ESPS failed with %d", retcode);
+ LM->Log(LM->msg);
+ }
+ }
+ else
+ {
+ espCQDUsed = TRUE;
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "PARALLEL_NUM_ESPS was set to %d", numPartitions);
+ LM->Log(LM->msg);
+ }
+ }
+ }
+ else if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "PARALLEL_NUM_ESPS not set; # partitions reported as %d", numPartitions);
+ LM->Log(LM->msg);
+ }
+
+ return espCQDUsed;
+}
+
+// Reset the cqd PARALLEL_NUM_ESPS. This is the other bookend for setEspParallelism(),
+// which returns TRUE if the cqd is actually set within that function. If so, this
+// function can be called to reset it.
+static void resetEspParallelism()
+{
+ HSLogMan *LM = HSLogMan::Instance();
+ Lng32 retcode = HSFuncExecQuery("CONTROL QUERY DEFAULT PARALLEL_NUM_ESPS RESET");
+ if (retcode)
+ {
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ {
+ HSLogMan *LM = HSLogMan::Instance();
+ sprintf(LM->msg, "cqd statement to reset PARALLEL_NUM_ESPS returned %d", retcode);
+ LM->Log(LM->msg);
+ }
+ }
+}
/**********************************************/
/* METHOD: createSampleOption() */
/* PURPOSE: create text for use in sample */
/* query. */
-/* RETCODE: 0 - on success */
-/* non-zero otherwise */
/* INPUT: sampleType - the sample option type*/
/* set when parsing. */
/* samplePercent - % */
@@ -3636,10 +3751,9 @@ void HSGlobalsClass::startJitLogging(const char* checkPointName, Int64 elapsedSe
/* when parsing. */
/* OUTPUT: sampleOpt - sample text option. */
/**********************************************/
-Lng32 createSampleOption(Lng32 sampleType, double samplePercent, NAString &sampleOpt,
+void createSampleOption(Lng32 sampleType, double samplePercent, NAString &sampleOpt,
Int64 sampleValue1, Int64 sampleValue2)
{
- Lng32 retcode = 0;
char floatStr[30], intStr[30];
switch (sampleType)
{
@@ -3671,10 +3785,9 @@ Lng32 createSampleOption(Lng32 sampleType, double samplePercent, NAString &sampl
sampleOpt += " ROWS ";
break;
default: // Invalid option.
- retcode = -1;
+ HS_ASSERT(FALSE);
break;
}
- return retcode;
}
@@ -3690,7 +3803,7 @@ void HSSample::makeTableName(NABoolean isPersSample)
NABoolean unpartitionedSample = FALSE;
if (objDef->getObjectFormat() == SQLMX)
{
- //The naming convention used for the temporary sample table is 'SQLMX_'
+ //The naming convention used for the temporary sample table is 'TRAF_SAMPLE_'
//followed by the object_uid of the source table and a portion of
//the timestamp. The object_uid ensures no collisions with update stats
//for other tables, while the timestamp chars help avoid collision when
@@ -3742,7 +3855,7 @@ void HSSample::makeTableName(NABoolean isPersSample)
convertInt64ToAscii(objDef->getObjectUID(), objectIDStr);
sprintf(timestampStr, "_%u_%u", (UInt32)tv.tv_sec, (UInt32)tv.tv_usec);
- sampleTable += "SQLMX_";
+ sampleTable += "TRAF_SAMPLE_";
sampleTable += objectIDStr;
sampleTable += timestampStr;
@@ -3789,7 +3902,6 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
Int64 &sampleRowCnt, // input/output
NABoolean isPersSample, // input. Default value is FALSE
NABoolean unpartitionedSample,// input. Default value is TRUE
- NABoolean createDandI,
Int64 minRowCtPerPartition
)
{
@@ -3812,10 +3924,10 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
// Create sample option based on sampling type, using 'samplePercent'.
if (hs_globals)
- retcode = createSampleOption(sampleType, samplePercent, sampleOption,
- hs_globals->sampleValue1, hs_globals->sampleValue2);
- else retcode = createSampleOption(sampleType, samplePercent, sampleOption);
- HSHandleError(retcode);
+ createSampleOption(sampleType, samplePercent, sampleOption,
+ hs_globals->sampleValue1, hs_globals->sampleValue2);
+ else
+ createSampleOption(sampleType, samplePercent, sampleOption);
//Normally, we want to create an AUDITED scratch table. Although, for
//performance and TMF timeouts, we should use a NON-AUDITED table and
@@ -3826,7 +3938,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
//AUTOABORT time. This process may take a long time and blow away
//your transaction.
LM->StartTimer("Create sample table");
- retcode = create(unpartitionedSample, isPersSample, createDandI);
+ retcode = create(unpartitionedSample, isPersSample);
LM->StopTimer();
if (retcode == -HS_PKEY_FLOAT_ERROR) {
// If creation of sample table fails with -1120, then the primary key
@@ -3851,19 +3963,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
// For Hive tables the sample table used is a Trafodion table
if (hs_globals->isHbaseTable || hs_globals->isHiveTable)
{
- // The optimal degree of parallelism for the LOAD or UPSERT is
- // the number of partitions of the original table. Force that.
- // Note that when the default for AGGRESSIVE_ESP_ALLOCATION_PER_CORE
- // is permanently changed to 'ON', we may be able to remove this CQD.
- if (hs_globals->objDef->getNumPartitions() > 1)
- {
- char temp[40]; // way more space than needed, but it's safe
- sprintf(temp,"'%d'",hs_globals->objDef->getNumPartitions());
- NAString EspsCQD = "CONTROL QUERY DEFAULT PARALLEL_NUM_ESPS ";
- EspsCQD += temp;
- HSFuncExecQuery(EspsCQD);
- EspCQDUsed = TRUE; // remember to reset later
- }
+ EspCQDUsed = setEspParallelism(hs_globals->objDef);
// Set CQDs controlling HBase cache size (number of rows returned by HBase
// in a batch) to avoid scanner timeout. Reset these after the sample query
@@ -4020,7 +4120,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
}
if (EspCQDUsed)
{
- HSFuncExecQuery("CONTROL QUERY DEFAULT PARALLEL_NUM_ESPS RESET");
+ resetEspParallelism();
}
if (retcode) TM->Rollback();
@@ -4038,6 +4138,7 @@ Lng32 HSSample::make(NABoolean rowCountIsEstimate, // input
if ((sampleRowCount == 0) && // sample set is empty;
(CmpCommon::getDefault(USTAT_USE_BULK_LOAD) == DF_OFF))
{ // cannot generate histograms
+ drop(); // drop the sample table we created
HSFuncMergeDiags(- UERR_SAMPLE_SET_IS_ZERO);
retcode = -1;
}
@@ -4471,6 +4572,7 @@ static void mapInternalSortTypes(HSColGroupStruct *groupList, NABoolean forHive
{
HSColumnStruct &col = group->colSet[0];
NAString columnName, dblQuote="\"";
+ group->ISSelectExpn = "";
// If retrieving from the Hive backing sample for a table, positional names
// are used. This avoids any issues with Trafodion delimited ids that do
@@ -4847,7 +4949,7 @@ void HSGlobalsClass::getMemoryRequirements(HSColGroupStruct* group, Int64 rows)
if (LM->LogNeeded())
{
- sprintf(LM->msg, "Memory estimates for signle group based on " PF64 " rows", rows);
+ sprintf(LM->msg, "Memory estimates for single group based on " PF64 " rows", rows);
LM->Log(LM->msg);
}
@@ -5103,7 +5205,8 @@ Lng32 HSGlobalsClass::CollectStatistics()
// Create a persistent sample table. It will be used for this non-IUS
// execution of Update Stats, and updated incrementally for subsequent
// IUS operations.
- HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName());
+ HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName(),
+ objDef->getSchemaName());
if (!sampleList)
return -1; // sample list didn't exist and failed creation
retcode = sampleList->createAndInsert(objDef,
@@ -5132,254 +5235,21 @@ Lng32 HSGlobalsClass::CollectStatistics()
{
return CollectStatisticsWithFastStats();
}
- else if ( canDoIUS() )
+ else if (canDoIUS())
{
- // Make sure the Where clause doesn't contain any constructs we don't allow
- // in the context of an IUS statement.
- retcode = validateIUSWhereClause();
- HSHandleError(retcode);
-
- ISMemPercentage_ = (float)CmpCommon::getDefaultNumeric(USTAT_IS_MEMORY_FRACTION);
-
- char ius_update_history_buffer[129];
- retcode = begin_IUS_work(ius_update_history_buffer);
+ // Use IUS and use output param 'done' to indicate if we need to carry
+ // on with RUS code below, or if IUS did it all and we can return.
+ NABoolean done = FALSE;
+ retcode = doIUS(done);
HSHandleError(retcode);
+ if (done)
+ return retcode;
- Int64 currentSampleSize = 0;
- Int64 futureSampleSize = 0;
-
- retcode = computeSampleSizeForIUS(currentSampleSize, futureSampleSize);
- HSHandleErrorIUS(retcode);
-
- // Setup the memory requirement for singlegroup in each memNeeded field,
- // Since singleGroup will receive the computed new stats, we compute
- // the memory needs by using futureSampleSize.
- mapInternalSortTypes(singleGroup);
- getMemoryRequirements(singleGroup, futureSampleSize);
-
- // =====================================================================
- // create the inMemory delete table, the cstr also setups the memory
- // requirement utilizing the currentSampleSize as #rows.
- // =====================================================================
- iusSampleDeletedInMem = new(STMTHEAP)
- HSInMemoryTable(*hssample_table,
- getWherePredicateForIUS(),
- currentSampleSize
- );
-
- // =====================================================================
- // Similarly, create the inMemory insert table.
- // =====================================================================
- NAString sampleTable_I(*hssample_table);
- sampleTable_I.append("_I");
-
- iusSampleInsertedInMem = new(STMTHEAP)
- HSInMemoryTable(sampleTable_I,
- getWherePredicateForIUS(),
- futureSampleSize,// worse case
- sampleRateAsPercetageForIUS);
-
- // =====================================================================
- // Find out which group has persistent CBFs and set the delayedRead flag
- // for it. Do it for all groups.
- // =====================================================================
- detectPersistentCBFsForIUS(*hssample_table, singleGroup);
-
-
- Int32 colsSelected = 0;
- NABoolean ranOutOfMem = FALSE;
-
- while ( moreColsForIUS() > 0 ) {
-
- // Select a set of columns for IUS, based on the availability
- // of memory. Selected columns will be marked in PENDING state.
- // The number of columns selected is returned.
- //
- // If a particular column has persistent CBF, its column
- // structure's delayedRead is set to TRUE.
- //
- retcode = selectIUSBatch(currentSampleSize, futureSampleSize, ranOutOfMem, colsSelected);
- HSHandleErrorIUS(retcode);
- checkTime("after selecting batch of columns for IUS");
-
- //
- // Require at least one column in the persistent sample table
- // to be read into memory in one batch
- //
- if ( colsSelected == 0 ) {
- if ( ranOutOfMem ) {
- diagsArea << DgSqlCode(UERR_IUS_NO_SUFFICIENT_MEMORY);
- retcode = -1;
- HSHandleErrorIUS(retcode);
- } else {
- if (LM->LogNeeded())
- LM->Log("Empty IUS batch, not because of memory.");
- }
- }
-
-
- // process column groups that are in PENDING state
- // Data from columns with delayedRead set to FALSE (i.e., no
- // corresponding persistent CBFs) will be read in.
- //
- // The rows to be deleted in one shot later on will be read in
- // from the sample table.
- //
- retcode = CollectStatisticsForIUS(currentSampleSize, futureSampleSize);
- HSHandleErrorIUS(retcode);
-
- //
- // Fall back to let internal sort based RUS to take care of any
- // groups failing to be updated via IUS. These groups are in PENDING
- // state. If all groups are processed, no more read and we are done!
- //
- HSColGroupStruct *group = singleGroup;
- Int32 cols = 0;
- Int32 colsToRead = 0;
-
- // First mask out those groups that already have data read in.
- // These groups should have delayedRead flag set to FALSE.
- //
- // We do need to properly merge the data from S(i-1), D and I
- // together, so that group->data points at the merged data and
- // group->nextdata points at the end+1 of the merged data.
- //
- // The merge algorithm:
- //
- // Allocate a temp. buffer of size (|S(i-1)| + |I|)
- // For all data item v in S(i-1) and I do
- // if ( v in cbf ) {
- // append data to the temp. buffer
- // remove one instance of v from cbf
- // }
- // delete group->data
- // set group->data = temp. buffer
- // set group->nextdata = temp. buffer's size + 1
-
- while (group) {
- if (group->state==PENDING) {
-
- if ( group->delayedRead == FALSE ) {
- group->state=SKIP;
- } else
- colsToRead++;
-
- cols++;
- }
- group = group->next;
- }
-
-
- if ( cols > 0 ) {
-
- if (LM->LogNeeded())
- LM->StartTimer("IUS: process failed groups with RUS");
-
- // First we need to read in column data if it has not been
- // read previously (i.e., those with delayedRead is TRUE, or
- // it has persistent CBF).
- if ( colsToRead > 0 )
- {
- HSCursor cursor;
- // Read from the persistent sample table and use smplGroup to
- // hold the data read. All columns in PENDING state
- // will be read in.
- retcode = readColumnsIntoMem(&cursor, currentSampleSize);
-
- HSHandleErrorIUS(retcode);
- checkTime("after reading pending columns from persistent sample table into memory for IUS->RUS reversion");
-
- }
-
- group = singleGroup;
- while (group) {
- if (group->state==SKIP) {
- group->state=PENDING;
- }
- group = group->next;
- }
-
- // Fill each group (in PENDING state)'s data area with data merged from
- // cbf, S(I-1) and I.
- retcode = mergeDatasetsForIUS();
- HSHandleErrorIUS(retcode);
- checkTime("after merging datasets for IUS->RUS reversion");
-
- group = singleGroup;
- while (group) {
- if (group->state==PENDING) {
-
- // Delete the histogram allocated for IUS. The RUS step
- // below will recompute groupHist.
- delete group->groupHist;
- group->groupHist = NULL;
- }
- group = group->next;
- }
-
-
- // Remove all persistent CBFs with PENDING state
- // because RUS may generates a histogram with different
- // #intervals than that recorded in CBF! If the CBFs
- // are left undeleted, the encoded intervals (as
- // # of buckets) in CBF can be in conflict with
- // the actual # of intervals computed by RUS.
- retcode = deletePersistentCBFsForIUS(*hssample_table, singleGroup);
- HSHandleErrorIUS(retcode);
-
- retcode = sortByColInMem();
- HSHandleErrorIUS(retcode);
-
- retcode = createStats(0 /* dummy argument */);
- HSHandleErrorIUS(retcode);
-
- if (LM->LogNeeded())
- LM->StopTimer();
- }
- }
-
-
-
- Int32 iusUnprocessed = 0;
- // Reverse the NO_STATS state to UNPROCESSED and count
- // total unprocessed
- HSColGroupStruct *group = singleGroup;
- while (group != NULL) {
- if (group->state == NO_STATS ) {
- group->state = UNPROCESSED;
- iusUnprocessed ++;
- } else
- if (group->state == UNPROCESSED )
- iusUnprocessed ++;
-
- group = group->next;
- }
-
-
-
- if ( iusUnprocessed > 0 ) {
- //
- // Fall back to the RUS code below when there is no sufficient memory to
- // process even one batch of columns, or there is no stats as a base to
- // compute IUS.
-
- // Remove all persistent CBFs with PENDING state
- // because RUS may generates a histogram with different
- // #intervals than that recorded in CBF! If the CBFs
- retcode = deletePersistentCBFsForIUS(*hssample_table, singleGroup);
- HSHandleErrorIUS(retcode);
-
-
- // setup the sample size
- sampleRowCount = futureSampleSize;
-
- } else {
-
- if (multiGroup) // don't return if there are MCs to process
- sampleRowCount = futureSampleSize;
- else
- return retcode;
- }
+ // Set sampling parameters to do an RUS corresponding to the existing
+ // persistent sample.
+ useSampling = TRUE;
+ externalSampleTable = TRUE;
+ sampleTblPercent = sampleRateAsPercetageForIUS * 100; // used for scaling results
}
NAString internalSortCQDValue = ActiveSchemaDB()->getDefaults().getValue(USTAT_INTERNAL_SORT);
@@ -5389,8 +5259,8 @@ Lng32 HSGlobalsClass::CollectStatistics()
// internal sort disabled
if (LM->LogNeeded()) LM->Log("Internal sort is disabled");
if (useSampling && !externalSampleTable)
- retcode = sampleTable.make(currentRowCountIsEstimate_,
- *hssample_table,
+ retcode = sampleTable.make(currentRowCountIsEstimate_,
+ *hssample_table,
actualRowCount, sampleRowCount);
// hssample_table assigned, actualRowCount and sampleRowCount may get adjusted.
else if (!externalSampleTable)
@@ -5510,10 +5380,9 @@ Lng32 HSGlobalsClass::CollectStatistics()
LM->Log("Internal sort: reading sample directly from base table; no sample table created");
*hssample_table = getTableName(user_table->data(), nameSpace);
// sampleTblPercent and sampleRowCount may get adjusted.
- retcode = createSampleOption(optFlags & SAMPLE_REQUESTED,
- sampleTblPercent, *sampleOption,
- sampleValue1, sampleValue2);
- HSHandleError(retcode);
+ createSampleOption(optFlags & SAMPLE_REQUESTED,
+ sampleTblPercent, *sampleOption,
+ sampleValue1, sampleValue2);
sampleTableUsed = FALSE;
samplingUsed = TRUE;
@@ -5526,8 +5395,8 @@ Lng32 HSGlobalsClass::CollectStatistics()
else
{
if (useSampling && !externalSampleTable)
- retcode = sampleTable.make(currentRowCountIsEstimate_,
- *hssample_table,
+ retcode = sampleTable.make(currentRowCountIsEstimate_,
+ *hssample_table,
actualRowCount, sampleRowCount);
// hssample_table assigned, actualRowCount and sampleRowCount may get adjusted.
else if (!externalSampleTable)
@@ -5831,6 +5700,311 @@ Lng32 HSGlobalsClass::CollectStatistics()
return retcode;
}
+// Do the setup work for IUS, and call either doFullIUS() or prepareToUsePersistentSample(),
+// depending on whether the USTAT_INCREMENTAL_UPDATE_STATISTICS cqd is ON or
+// SAMPLE. In the latter case prepareToUsePersistentSample() is called to update
+// the persistent sample incrementally, and then the normal Update Stats algorithm
+// executes using the persistent sample table.
+//
+// The 'done' parameter is returned with a value of TRUE if stats are completely
+// handled by doFullIUS(). If prepareToUsePersistentSample() is called instead,
+// or if doFullIUS() is unable to incrementally update the stats for one or more
+// columns (e.g., shape test failure), 'done' will be set to FALSE.
+Lng32 HSGlobalsClass::doIUS(NABoolean& done)
+{
+ done = FALSE; // set to TRUE if IUS successfully updates the stats for all columns
+ Lng32 retcode = 0;
+
+ // Make sure the Where clause doesn't contain any constructs we don't allow
+ // in the context of an IUS statement.
+ retcode = validateIUSWhereClause();
+ HSHandleError(retcode);
+
+ retcode = begin_IUS_work();
+ HSHandleError(retcode);
+
+ Int64 currentSampleSize = 0;
+ Int64 futureSampleSize = 0;
+
+ retcode = computeSampleSizeForIUS(currentSampleSize, futureSampleSize);
+ HSHandleErrorIUS(retcode);
+
+ DefaultToken iusOption = CmpCommon::getDefault(USTAT_INCREMENTAL_UPDATE_STATISTICS);
+ if (iusOption == DF_ON)
+ return doFullIUS(currentSampleSize, futureSampleSize, done);
+ else if (iusOption == DF_SAMPLE)
+ // Leave 'done' FALSE; prepareToUsePersistentSample() updates the persistent sample
+ // table in preparation for use by RUS.
+ return prepareToUsePersistentSample(currentSampleSize, futureSampleSize);
+ else
+ {
+ // Exception will be thrown, ~HSGlobalsClass will call end_IUS_work().
+ HS_ASSERT(false);
+ return -1; // avoid 'no return' warning
+ }
+}
+
+// Try to incrementally update existing histograms using in-memory tables and
+// CBFs. If reversion to RUS is required for one or more columns, the 'done'
+// output parameter will be set to FALSE.
+Lng32 HSGlobalsClass::doFullIUS(Int64 currentSampleSize,
+ Int64 futureSampleSize,
+ NABoolean& done)
+{
+ done = FALSE; // unless IUS handles all columns
+ HSLogMan* LM = HSLogMan::Instance();
+ Lng32 retcode = 0;
+ ISMemPercentage_ = (float)CmpCommon::getDefaultNumeric(USTAT_IS_MEMORY_FRACTION);
+
+ // Setup the memory requirement for singlegroup in each memNeeded field,
+ // Since singleGroup will receive the computed new stats, we compute
+ // the memory needs by using futureSampleSize.
+ mapInternalSortTypes(singleGroup);
+ getMemoryRequirements(singleGroup, futureSampleSize);
+
+ // =====================================================================
+ // create the inMemory delete table, the cstr also setups the memory
+ // requirement utilizing the currentSampleSize as #rows.
+ // =====================================================================
+ iusSampleDeletedInMem = new(STMTHEAP)
+ HSInMemoryTable(*hssample_table,
+ getWherePredicateForIUS(),
+ currentSampleSize
+ );
+
+ // =====================================================================
+ // Similarly, create the inMemory insert table.
+ // =====================================================================
+ NAString sampleTable_I(*hssample_table);
+ sampleTable_I.append("_I");
+
+ iusSampleInsertedInMem = new(STMTHEAP)
+ HSInMemoryTable(sampleTable_I,
+ getWherePredicateForIUS(),
+ futureSampleSize,// worse case
+ sampleRateAsPercetageForIUS);
+
+ // =====================================================================
+ // Find out which group has persistent CBFs and set the delayedRead flag
+ // for it. Do it for all groups.
+ // =====================================================================
+ detectPersistentCBFsForIUS(*hssample_table, singleGroup);
+
+
+ Int32 colsSelected = 0;
+ NABoolean ranOutOfMem = FALSE;
+
+ while ( moreColsForIUS() > 0 ) {
+
+ // Select a set of columns for IUS, based on the availability
+ // of memory. Selected columns will be marked in PENDING state.
+ // The number of columns selected is returned.
+ //
+ // If a particular column has persistent CBF, its column
+ // structure's delayedRead is set to TRUE.
+ //
+ retcode = selectIUSBatch(currentSampleSize, futureSampleSize, ranOutOfMem, colsSelected);
+ HSHandleErrorIUS(retcode);
+ checkTime("after selecting batch of columns for IUS");
+
+ //
+ // Require at least one column in the persistent sample table
+ // to be read into memory in one batch
+ //
+ if ( colsSelected == 0 ) {
+ if ( ranOutOfMem ) {
+ diagsArea << DgSqlCode(UERR_WARNING_IUS_INSUFFICIENT_MEMORY)
+ << DgInt0(moreColsForIUS());
+ break; // Let RUS handle the rest
+ } else {
+ if (LM->LogNeeded())
+ LM->Log("Empty IUS batch, not because of memory.");
+ }
+ }
+
+
+ // process column groups that are in PENDING state
+ // Data from columns with delayedRead set to FALSE (i.e., no
+ // corresponding persistent CBFs) will be read in.
+ //
+ // The rows to be deleted in one shot later on will be read in
+ // from the sample table.
+ //
+ retcode = CollectStatisticsForIUS(currentSampleSize, futureSampleSize);
+ HSHandleErrorIUS(retcode);
+
+ //
+ // Fall back to let internal sort based RUS to take care of any
+ // groups failing to be updated via IUS. These groups are in PENDING
+ // state. If all groups are processed, no more read and we are done!
+ //
+ HSColGroupStruct *group = singleGroup;
+ Int32 cols = 0;
+ Int32 colsToRead = 0;
+
+ // First mask out those groups that already have data read in.
+ // These groups should have delayedRead flag set to FALSE.
+ //
+ // We do need to properly merge the data from S(i-1), D and I
+ // together, so that group->data points at the merged data and
+ // group->nextdata points at the end+1 of the merged data.
+ //
+ // The merge algorithm:
+ //
+ // Allocate a temp. buffer of size (|S(i-1)| + |I|)
+ // For all data item v in S(i-1) and I do
+ // if ( v in cbf ) {
+ // append data to the temp. buffer
+ // remove one instance of v from cbf
+ // }
+ // delete group->data
+ // set group->data = temp. buffer
+ // set group->nextdata = temp. buffer's size + 1
+
+ while (group) {
+ if (group->state==PENDING) {
+
+ if ( group->delayedRead == FALSE ) {
+ group->state=SKIP;
+ } else
+ colsToRead++;
+
+ cols++;
+ }
+ group = group->next;
+ }
+
+
+ if ( cols > 0 ) {
+
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: process failed groups with RUS");
+
+ // First we need to read in column data if it has not been
+ // read previously (i.e., those with delayedRead is TRUE, or
+ // it has persistent CBF).
+ if ( colsToRead > 0 )
+ {
+ HSCursor cursor;
+ // Read from the persistent sample table and use smplGroup to
+ // hold the data read. All columns in PENDING state
+ // will be read in.
+ retcode = readColumnsIntoMem(&cursor, currentSampleSize);
+
+ HSHandleErrorIUS(retcode);
+ checkTime("after reading pending columns from persistent sample table into memory for IUS->RUS reversion");
+
+ }
+
+ group = singleGroup;
+ while (group) {
+ if (group->state==SKIP) {
+ group->state=PENDING;
+ }
+ group = group->next;
+ }
+
+ // Fill each group (in PENDING state)'s data area with data merged from
+ // cbf, S(I-1) and I.
+ retcode = mergeDatasetsForIUS();
+ HSHandleErrorIUS(retcode);
+ checkTime("after merging datasets for IUS->RUS reversion");
+
+ group = singleGroup;
+ while (group) {
+ if (group->state==PENDING) {
+
+ // Delete the histogram allocated for IUS. The RUS step
+ // below will recompute groupHist.
+ delete group->groupHist;
+ group->groupHist = NULL;
+ }
+ group = group->next;
+ }
+
+
+ // Remove all persistent CBFs with PENDING state
+ // because RUS may generates a histogram with different
+ // #intervals than that recorded in CBF! If the CBFs
+ // are left undeleted, the encoded intervals (as
+ // # of buckets) in CBF can be in conflict with
+ // the actual # of intervals computed by RUS.
+ retcode = deletePersistentCBFsForIUS(*hssample_table, singleGroup, PENDING);
+ HSHandleErrorIUS(retcode);
+
+ retcode = sortByColInMem();
+ HSHandleErrorIUS(retcode);
+
+ retcode = createStats(0 /* dummy argument */);
+ HSHandleErrorIUS(retcode);
+
+ if (LM->LogNeeded())
+ LM->StopTimer();
+ }
+ } // while ( moreColsForIUS() > 0 )
+
+ // The _I table can be dropped after using it to update the persistent sample
+ // table, which must be done before doing RUS on any unprocessed columns (RUS
+ // will use the updated persistent sample).
+ retcode = UpdateIUSPersistentSampleTable(currentSampleSize, futureSampleSize, sampleRowCount);
+ HSHandleErrorIUS(retcode);
+ if (sampleIExists_) {
+ retcode = drop_I(*hssample_table);
+ HSHandleErrorIUS(retcode);
+ sampleIExists_ = FALSE; // only try to drop it once
+ }
+
+ Int32 iusUnprocessed = 0;
+ // Reverse the NO_STATS state to UNPROCESSED and count
+ // total unprocessed
+ HSColGroupStruct *group = singleGroup;
+ while (group != NULL) {
+ if (group->state == NO_STATS ) {
+ group->state = UNPROCESSED;
+ iusUnprocessed ++;
+ } else
+ if (group->state == UNPROCESSED )
+ iusUnprocessed ++;
+
+ group = group->next;
+ }
+
+ // Leave the 'done' parameter FALSE so we continue with the RUS code upon return
+ // if there are unprocessed columns (not enough memory or no prior stats as a
+ // base to compute IUS), or if there are MCs to process.
+ if ( iusUnprocessed > 0 ) {
+ // Remove all persistent CBFs with UNPROCESSED state because RUS may generate
+ // a histogram with different #intervals than that recorded in CBF! If the CBFs
+ // are left undeleted, the encoded intervals (as # of buckets) in CBF can be in
+ // conflict with the actual # of intervals computed by RUS.
+ retcode = deletePersistentCBFsForIUS(*hssample_table, singleGroup, UNPROCESSED);
+ HSHandleErrorIUS(retcode);
+ } else if (multiGroup) { // not done if there are MCs to process
+ } else {
+ done = TRUE; // no need to use RUS code
+ }
+
+ return retcode;
+}
+
+// This function makes all preparations for doing RUS using an updated IUS
+// persistent sample table. The sample table is updated, and obsolete CBFs
+// are discarded,
+Lng32 HSGlobalsClass::prepareToUsePersistentSample(Int64 currentSampleSize,
+ Int64 futureSampleSize)
+{
+ Lng32 retcode = 0;
+ retcode = UpdateIUSPersistentSampleTable(currentSampleSize, futureSampleSize, sampleRowCount);
+ HSHandleErrorIUS(retcode);
+
+ // If there are existing CBFs, they will be obsolete once the current operation
+ // completes.
+ retcode = deletePersistentCBFsForIUS(*hssample_table, singleGroup, UNPROCESSED);
+ HSHandleErrorIUS(retcode);
+
+ return retcode;
+}
+
//
// A help function to generate a SQL timestamp constant. Example: '2012-01-01 23:59:00'
//
@@ -5896,7 +6070,7 @@ void genArkcmpInfo(NAString& nidpid)
// against a target table. The transaction is established by updating the row in the
// PERSISTENT_SAMPLES table about the persistent sample table used by the IUS:
// 1. UPDATE_DATE field is populated with the current timestamp;
-// 2. IUS_UPDATE_HISTORY field is populated with the SQ node ID and process ID
+// 2. UPDATER_INFO field is populated with the SQ node ID and process ID
// of the tdm_arkcmp process performing the IUS work
// The method will return 0 after the above successful updates, indicating that the IUS
// work can proceed.
@@ -5909,8 +6083,8 @@ void genArkcmpInfo(NAString& nidpid)
// CQD(USTAT_IUS_MAX_TRANSACTION_DURATION)), the ongoing transaction is
// considered legitimate, and the current call to the method will return an error
// indicating that a concurrent IUS is in progress.
-// The argument ius_update_history_buffer will be filled with the string
-// read from the corresponding field IUS_UPDATE_HISTORY.
+// ius_update_history_buffer will be filled with the string read from the
+// UPDATER_INFO column.
//
// When P2-P1 > CQD(USTAT_IUS_MAX_TRANSACTION_DURATION), the on-going transaction is
// considered over-due and will be discarded. The method proceeds as if there was
@@ -5923,14 +6097,17 @@ void genArkcmpInfo(NAString& nidpid)
// The CQD USTAT_IUS_MAX_TRANSACTION_DURATION specifies the max transaction
// duration allowed with the unit in minutes. The default value is 720 minutes (12 hours).
-Lng32 HSGlobalsClass::begin_IUS_work(char* ius_update_history_buffer)
+Lng32 HSGlobalsClass::begin_IUS_work()
{
+ sampleIExists_ = FALSE; // keep track of whether a _I table needs to be dropped
+
#ifdef _DEBUG
if (CmpCommon::getDefault(USTAT_IUS_NO_BLOCK) == DF_ON)
return 0;
-#endif
+#endif
- HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName());
+ HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName(),
+ objDef->getSchemaName());
if ( !sampleList ) return -1;
Int64 updTimestamp = 0;
@@ -5938,11 +6115,20 @@ Lng32 HSGlobalsClass::begin_IUS_work(char* ius_update_history_buffer)
HSTranMan *TM = HSTranMan::Instance();
TM->Begin("READ AND UPDATE THE UPDATE DATE AND HISTORY from PERSISTENT SAMPLE TABLE");
-
+ char ius_update_history_buffer[129];
Lng32 retcode =
sampleList->readIUSUpdateInfo(objDef, ius_update_history_buffer, &updTimestamp);
- if (retcode != 100) // need to carry on if no row for this table
- HSHandleError(retcode);
+ if (retcode == 100)
+ {
+ HSFuncMergeDiags(- UERR_IUS_NO_PERSISTENT_SAMPLE,
+ objDef->getObjectFullName().data());
+ retcode = -1;
+ HSHandleError(retcode);
+ }
+ else
+ {
+ HSHandleError(retcode);
+ }
time_t t;
@@ -5988,19 +6174,23 @@ Lng32 HSGlobalsClass::begin_IUS_work(char* ius_update_history_buffer)
retcode =
sampleList->updIUSUpdateInfo(objDef,
(char*)nid_pid_str.data(),
- (char*)updTimestampStr.data());
+ (char*)updTimestampStr.data(),
+ 0 /* don't write where condition now */);
- if (retcode)
- TM->Rollback();
- else
- TM->Commit();
+ if (retcode == 100)
+ {
+ HSFuncMergeDiags(- UERR_IUS_NO_PERSISTENT_SAMPLE,
+ objDef->getObjectFullName().data());
+ retcode = -1;
+ }
HSHandleError(retcode);
//
- // If we reach here, it means we have successfully stored the bdt and our process info
- // into the UPDATE_DATE and IUS_UPDATE_HISTORY column. We can return TRUE.
+ // If we reach here, we have successfully stored the bdt and our process info
+ // into the UPDATE_DATE and UPDATER_INFO columns of SB_PERSISTENT_SAMPLES.
//
+ PSRowUpdated = TRUE;
return 0;
}
@@ -6010,16 +6200,25 @@ Lng32 HSGlobalsClass::begin_IUS_work(char* ius_update_history_buffer)
// as follows.
//
// 1. UPDATE_DATE field is reset to a timestamp representing epoch time;
-// 2. IUS_UPDATE_HISTORY field is reset to an empty string.
+// 2. UPDATER_INFO field is reset to an empty string.
Lng32 HSGlobalsClass::end_IUS_work()
{
+ Lng32 retcode = 0;
+ if (sampleIExists_)
+ {
+ retcode = drop_I(*hssample_table);
+ // ignore retcode; we want to try the rest of this method as well
+ sampleIExists_ = FALSE;
+ }
+
#ifdef _DEBUG
if (CmpCommon::getDefault(USTAT_IUS_NO_BLOCK) == DF_ON)
return 0;
#endif
- HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName());
+ HSPersSamples *sampleList = HSPersSamples::Instance(objDef->getCatName(),
+ objDef->getSchemaName());
if ( !sampleList ) return -1;
// The epoch time
@@ -6032,8 +6231,13 @@ Lng32 HSGlobalsClass::end_IUS_work()
NAString updTimestampStr;
genSQLTimestampConstant(bdt, updTimestampStr);
- Lng32 retcode =
- sampleList->updIUSUpdateInfo(objDef, (char*)"", (char*)updTimestampStr.data());
+ retcode =
+ sampleList->updIUSUpdateInfo(objDef,
+ (char*)"",
+ (char*)updTimestampStr.data(),
+ getWherePredicateForIUS(),
+ PST_IUSrequestedSampleRows_,
+ PST_IUSactualSampleRows_);
HSHandleError(retcode);
return 0;
@@ -6104,35 +6308,36 @@ Lng32 HSGlobalsClass::computeSampleSizeForIUS(Int64& currentSampleSize, Int64& f
return 0;
}
+// Before we have the PERSISTENT_DATA table available to us, we will
+// save the CBFs as binary files on disk. One CBF maps to one binary file.
+// The path of the directory for these files is specified in CQD
+// USTAT_IUS_PERSISTENT_CBF_PATH, and the cbf file name is
+// sampleTableName + '.' + 'colName'. This function builds the common initial
+// text of the path for all columns in the same table, and assigns it to the
+// output parameter filePrefix.
+void HSGlobalsClass::getCBFFilePrefix(NAString& sampleTableName, NAString& filePrefix)
+{
+ filePrefix = ActiveSchemaDB()->getDefaults().getValue(USTAT_IUS_PERSISTENT_CBF_PATH);
+ filePrefix.append("/")
+ .append(sampleTableName)
+ .append(".");
+}
void
HSGlobalsClass::detectPersistentCBFsForIUS(NAString& sampleTableName,
HSColGroupStruct *group)
{
- // Before we have the PERSISTENT_DATA table available to us, we will
- // save the CBFs as binary files on disk. One CBF maps to one binary file.
- // The path of the directory for these files is specified in CQD
- // USTAT_IUS_PERSISTENT_CBF_PATH, and the cbf binary file name is
- // sampleTableName + '.' + 'colName';
-
- NAString path =
- ActiveSchemaDB()->getDefaults().getValue(USTAT_IUS_PERSISTENT_CBF_PATH);
-
- NAString cbfFilePrefix(path);
- cbfFilePrefix.append("/");
- cbfFilePrefix.append(sampleTableName);
- cbfFilePrefix.append(".");
-
+ NAString cbfFilePrefix;
+ getCBFFilePrefix(sampleTableName, cbfFilePrefix);
struct stat sts;
while (group) {
-
NAString cbfFile(cbfFilePrefix);
- cbfFile.append(group->colSet[0].colname->data());
+ cbfFile.append(group->cbfFileNameSuffix());
if (stat(cbfFile, &sts) == -1 && errno == ENOENT)
group->delayedRead = FALSE;
else
- group->delayedRead = TRUE;
+ group->delayedRead = TRUE;
group = group->next;
}
@@ -6197,7 +6402,7 @@ Lng32 HSGlobalsClass::prepareForIUSAlgorithm1(Int64& rows)
// Populate the deleted rows into a in-memory table.
NAString delQuery;
- iusSampleDeletedInMem->generateDeleteQuery(*hssample_table, delQuery);
+ generateIUSDeleteQuery(*hssample_table, delQuery);
@@ -6266,6 +6471,27 @@ Lng32 HSGlobalsClass::prepareForIUSAlgorithm1(Int64& rows)
return 0;
}
+static Lng32 create_I(NAString& sampTblName)
+{
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: create _I table");
+
+ NAString createI("create table ");
+ createI += sampTblName;
+ createI += "_I LIKE ";
+ createI += sampTblName;
+ createI += " WITH PARTITIONS";
+ Lng32 retcode = HSFuncExecQuery(createI, -UERR_INTERNAL_ERROR,
+ NULL, "IUS create I",
+ NULL, NULL, TRUE/*doRetry*/);
+ if (LM->LogNeeded())
+ LM->StopTimer();
+
+ HSHandleError(retcode);
+ return retcode;
+}
+
Lng32 HSGlobalsClass::generateSampleI(Int64 currentSampleSize,
Int64 futureSampleSize,
@@ -6280,11 +6506,11 @@ Lng32 HSGlobalsClass::generateSampleI(Int64 currentSampleSize,
if (LM->LogNeeded())
LM->StartTimer("IUS: select-insert data set I");
- // performing:
- //
- // insert into <sample_I>
- // (select * from <sourceTable> where <where> sample);
- //
+ // performing:
+ //
+ // upsert using load into <sample_I>
+ // (select * from <sourceTable> where <where> sample);
+ //
NAString sampleTable_I(*hssample_table);
sampleTable_I.append("_I");
@@ -6292,18 +6518,20 @@ Lng32 HSGlobalsClass::generateSampleI(Int64 currentSampleSize,
HSTranMan *TM = HSTranMan::Instance();
NABoolean transStarted = (TM->Begin("IUS clean data set I") == 0);
- if (LM->LogNeeded()) LM->StartTimer("IUS: select-insert to form persistent I");
-
-
NAString insertSelectIQuery;
iusSampleInsertedInMem->generateInsertSelectIQuery(sampleTable_I,
*user_table, insertSelectIQuery,
currentSampleSize, futureSampleSize,
deleteSetSize, actualRowCount);
+ NABoolean needEspParReset = setEspParallelism(objDef);
retcode = HSFuncExecQuery(insertSelectIQuery, -UERR_INTERNAL_ERROR, &xRows,
"IUS data set I creation",
- NULL, NULL, TRUE/*doRetry*/ );
+ NULL, NULL, TRUE/*doRetry*/,
+ 0, TRUE); // check for MDAM usage
+
+ if (needEspParReset)
+ resetEspParallelism();
if (retcode) TM->Rollback();
@@ -6323,22 +6551,17 @@ Lng32 HSGlobalsClass::generateSampleI(Int64 currentSampleSize,
static Lng32 drop_I(NAString& sampTblName)
{
- NAString cleanupI("drop table ");
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: drop _I table");
+
+ NAString cleanupI("drop table if exists ");
cleanupI.append(sampTblName).append("_I");
Lng32 retcode = HSFuncExecQuery(cleanupI, -UERR_INTERNAL_ERROR,
NULL, "IUS cleanup I",
NULL, NULL, TRUE/*doRetry*/);
- HSHandleError(retcode);
- return retcode;
-}
-
-static Lng32 drop_D(NAString& sampTblName)
-{
- NAString cleanupI("drop table ");
- cleanupI.append(sampTblName).append("_D");
- Lng32 retcode = HSFuncExecQuery(cleanupI, -UERR_INTERNAL_ERROR,
- NULL, "IUS cleanup D",
- NULL, NULL, TRUE/*doRetry*/);
+ if (LM->LogNeeded())
+ LM->StopTimer();
HSHandleError(retcode);
return retcode;
}
@@ -6346,49 +6569,54 @@ static Lng32 drop_D(NAString& sampTblName)
Lng32 HSGlobalsClass::CollectStatisticsForIUS(Int64 currentSampleSize,
Int64 futureSampleSIze)
{
- Lng32 retcode = 0;
- Int64 xRows = 0;
-
- HSLogMan *LM = HSLogMan::Instance();
-
- // create help table -I and -D
- {
- diagsArea << DgSqlCode(-4222)
- << DgString0("IUS Sample Table");
- }
-
+ Lng32 retcode = 0;
+ Int64 xRows = 0;
- if (LM->LogNeeded())
- LM->StartTimer("IUS: read in Si");
+ HSLogMan *LM = HSLogMan::Instance();
- HSColGroupStruct *group = singleGroup;
- while (group) {
- if (group->delayedRead && group->state == PENDING)
- group->state = SKIP ; // temp. set so that the column
- // data is not to be read.
- group = group->next;
- }
+ // create help table _I
+ if (!sampleIExists_)
+ {
+ retcode = create_I(*hssample_table);
+ HSHandleError(retcode);
+ sampleIExists_ = TRUE; // so we remember to drop it
+ }
+ NABoolean havePending = FALSE;
+ HSColGroupStruct *group = singleGroup;
+ while (group)
+ {
+ if (group->state == PENDING)
+ {
+ if (group->delayedRead)
+ group->state = SKIP; // temp. set so that the column data is not to be read
+ else
+ havePending = TRUE;
+ }
+ group = group->next;
+ }
- // Populate the selected rows into singleGroup. Use a C scope to allow
- // the cursor to be deallocated after the reading of S.
- //
- // Only read in columns that are in PENDING state
- {
- HSCursor cursor;
- // Read from the persistent sample table and use smplGroup to
- // hold the data read. All columns in PENDING state
- // will be read in.
- retcode = readColumnsIntoMem(&cursor, currentSampleSize);
- HSHandleError(retcode);
- checkTime("after reading pending columns into memory for IUS");
- }
+ // Only read in Si if there is at least one column in this batch that doesn't
+ // already have a persistent CBF.
+ if (havePending)
+ {
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: read in Si");
+ // Populate the data areas for the PENDING columns from the persistent
+ // sample table.
+ HSCursor cursor; // on block exit, dtor will close/dealloc stmt and descriptors
+ retcode = readColumnsIntoMem(&cursor, currentSampleSize);
+ HSHandleError(retcode);
+ checkTime("after reading pending columns into memory for IUS");
- if (LM->LogNeeded())
- LM->StopTimer();
+ if (LM->LogNeeded())
+ LM->StopTimer();
+ }
+ else if (LM->LogNeeded())
+ LM->Log("IUS: skipped reading in Si; delayedRead true for all columns in batch");
- // restore the state field to PENDING.
+ // restore the state field to PENDING for any skipped groups.
group = singleGroup;
while (group) {
if (group->delayedRead && group->state == SKIP)
@@ -6415,6 +6643,7 @@ Lng32 HSGlobalsClass::CollectStatisticsForIUS(Int64 currentSampleSize,
selectDQuery);
retcode = iusSampleDeletedInMem->populate(selectDQuery);
+ HSHandleError(retcode);
if (LM->LogNeeded())
LM->StopTimer();
@@ -6466,63 +6695,119 @@ Lng32 HSGlobalsClass::CollectStatisticsForIUS(Int64 currentSampleSize,
if (LM->LogNeeded())
LM->StopTimer();
-
- // Update the persistent sample table in several steps
- {
- HSFuncExecQuery("CONTROL QUERY DEFAULT ALLOW_DML_ON_NONAUDITED_TABLE 'ON'");
-
- // start a new scope for the trasaction
- HSTranController TC("IUS: update PS table", &retcode);
- HSHandleError(retcode);
+ // Write CBFs for groups that are PROCESSED and delayedRead back to disk.
+ writeCBFstoDiskForIUS(*hssample_table, singleGroup);
- // step 1 - delete the affected rows from PS
- NAString deleteQuery;
- iusSampleDeletedInMem->generateDeleteQuery(*hssample_table, deleteQuery);
+ return retcode;
+}
+// Returns the table component of the fully qualified name passed in, using the
+// catalog and schema name from tblDef to determine where it starts (the table
+// is in the same schema as the one referenced by tblDef). This avoids problems
+// in parsing the fully qualified name posed by the possibility of periods within
+// delimited identifiers.
+static const char* extractTblName(const NAString& fullyQualifiedName,
+ HSTableDef* tblDef)
+{
+ Lng32 tblNameOffset = tblDef->getCatName().length() +
+ tblDef->getSchemaName().length() +
+ 2; // 2 dot separators
+ return fullyQualifiedName.data() + tblNameOffset;
+}
+
+// Update the persistent sample table and determine its new cardinality.
+// 1) Delete rows in the persistent sample satisfying the IUS predicate.
+// 2) Insert the rows from <sampleTblName>_I into the persistent sample
+// if the _I table was created. If building histograms from scratch
+// using the persistent sample rather than incrementally changing them
+// (USTAT_INCREMENTAL_UPDATE_STATISTICS is DF_SAMPLE), insert sampled
+// rows satisfying the IUS where clause directly from the source table
+// In either case, these rows constitute a random sample of rows from
+// the source table that satisfy the IUS predicate.
+// 3) From the prior cardinality of the sample table (oldSampleSize), subtract
+// the number of rows deleted, add the number of rows inserted, and return
+// the result in the newSampleSize parameter.
+//
+// This can't be done as part of end_IUS_work(), because that is called even
+// when the IUS fails; its purpose is just to modify the SB_PERSISTENT_SAMPLES
+// table to indicate that IUS is no longer in progress on the source table.
+// The persistent sample table itself is only modified if IUS is successful.
+Lng32 HSGlobalsClass::UpdateIUSPersistentSampleTable(Int64 oldSampleSize,
+ Int64 requestedSampleSize,
+ Int64& newSampleSize)
+{
+ Lng32 retcode = 0;
+ Int64 rowsAffected;
+ HSLogMan *LM = HSLogMan::Instance();
+ newSampleSize = oldSampleSize; // before deleting/adding rows
- if (LM->LogNeeded()) {
- LM->Log("query to delete from PS:");
- LM->Log(deleteQuery.data());
- }
+ HSFuncExecQuery("CONTROL QUERY DEFAULT ALLOW_DML_ON_NONAUDITED_TABLE 'ON'");
+ HSTranController TC("IUS: update PS table", &retcode);
+ HSHandleError(retcode);
- retcode = HSFuncExecQuery(deleteQuery, -UERR_INTERNAL_ERROR,
- &xRows,
- "IUS delete from PS where",
- NULL, NULL, TRUE/*doRetry*/ );
- HSHandleError(retcode);
+ // step 1 - delete the affected rows from PS
+ NAString deleteQuery;
+ generateIUSDeleteQuery(*hssample_table, deleteQuery);
- // step 2 - add all rows from _I to PS
- NAString selectInsertQuery;
- iusSampleInsertedInMem->generateSelectInsertQuery(*hssample_table, *user_table, selectInsertQuery);
+ if (LM->LogNeeded()) {
+ LM->Log("query to delete from PS:");
+ LM->Log(deleteQuery.data());
+ LM->StartTimer("IUS: execute query to delete from PS");
+ }
- if (LM->LogNeeded()) {
- LM->Log("query to insert into PS:");
- LM->Log(selectInsertQuery.data());
- }
-
- retcode = HSFuncExecQuery(selectInsertQuery, -UERR_INTERNAL_ERROR,
- &xRows,
- "IUS insert into PS (select from _I)",
- NULL, NULL, TRUE/*doRetry*/ );
- HSHandleError(retcode);
+ rowsAffected = 0;
+ retcode = HSFuncExecQuery(deleteQuery, -UERR_INTERNAL_ERROR,
+ &rowsAffected,
+ "IUS delete from PS where",
+ NULL, NULL, TRUE/*doRetry*/ );
+ if (LM->LogNeeded()) {
+ LM->StopTimer();
+ sprintf(LM->msg, PF64 " rows deleted from persistent sample table.", rowsAffected);
+ LM->Log(LM->msg);
+ }
+ HSHandleError(retcode);
+ newSampleSize -= rowsAffected;
- // step3 - drop _I table
- retcode = drop_I(*hssample_table);
- HSHandleError(retcode);
+ // step 2 - add all rows from _I to PS, or sampled from source table,
+ // depending on USTAT_INCREMENTAL_UPDATE_STATISTICS value.
+ NAString selectInsertQuery;
+ generateIUSSelectInsertQuery(*hssample_table, *user_table, selectInsertQuery);
- // step4 - drop _D table
- retcode = drop_D(*hssample_table);
- HSHandleError(retcode);
+ if (LM->LogNeeded()) {
+ LM->Log("query to insert into PS:");
+ LM->Log(selectInsertQuery.data());
+ LM->StartTimer("IUS: execute query to insert into PS");
+ }
- HSFuncExecQuery("CONTROL QUERY DEFAULT ALLOW_DML_ON_NONAUDITED_TABLE reset");
+ rowsAffected = 0;
+ const char* insSourceTblName = extractTblName(*hssample_table + "_I", objDef);
+ NABoolean needEspParReset = setEspParallelism(objDef, insSourceTblName);
+ retcode = HSFuncExecQuery(selectInsertQuery, -UERR_INTERNAL_ERROR,
+ &rowsAffected,
+ "IUS insert into PS (select from _I)",
+ NULL, NULL, TRUE/*doRetry*/, 0,
+ // check mdam usage if reading incremental sample directly from source table
+ CmpCommon::getDefault(USTAT_INCREMENTAL_UPDATE_STATISTICS) == DF_SAMPLE); //checkMdam
+ if (LM->LogNeeded()) {
+ LM->StopTimer();
+ sprintf(LM->msg, PF64 " rows inserted into persistent sample table.", rowsAffected);
+ LM->Log(LM->msg);
}
+ if (needEspParReset)
+ resetEspParallelism();
+ HSHandleError(retcode);
+ newSampleSize += rowsAffected;
- checkTime("after updating persistent sample table for IUS");
+ // Save sample count values to update row in SB_PERSISTENT_SAMPLES table.
+ PST_IUSrequestedSampleRows_ = new(STMTHEAP) Int64;
+ *PST_IUSrequestedSampleRows_ = requestedSampleSize;
+ PST_IUSactualSampleRows_ = new(STMTHEAP) Int64;
+ *PST_IUSactualSampleRows_ = newSampleSize;
- // Write CBFs for groups that are PENDING and delayedRead back to disk.
- Int32 count = writeCBFstoDiskForIUS(*hssample_table,singleGroup);
+ HSFuncExecQuery("CONTROL QUERY DEFAULT ALLOW_DML_ON_NONAUDITED_TABLE reset");
+ checkTime("after updating persistent sample table for IUS");
return retcode;
}
@@ -6531,25 +6816,13 @@ Int32 HSGlobalsClass::readCBFsIntoMemForIUS(NAString& sampleTableName,
HSColGroupStruct* group
)
{
- // Before we have the PERSISTENT_DATA table available to us, we will
- // save the CBFs as binary files on disk. One CBF maps to one binary file.
- // The path of the directory for these files is specified in CQD
- // USTAT_IUS_PERSISTENT_CBF_PATH, and the cbf binary file name is
- // sampleTableName + '.' + 'colName';
-
- NAString path =
- ActiveSchemaDB()->getDefaults().getValue(USTAT_IUS_PERSISTENT_CBF_PATH);
-
- NAString cbfFilePrefix(path);
- cbfFilePrefix.append("/");
- cbfFilePrefix.append(sampleTableName);
- cbfFilePrefix.append(".");
+ NAString cbfFilePrefix;
+ getCBFFilePrefix(sampleTableName, cbfFilePrefix);
Lng32 sz;
Lng32 bufSz = 0;
char* bufptr = NULL;
struct stat sts;
- char buffer[20];
while (group) {
@@ -6560,8 +6833,7 @@ Int32 HSGlobalsClass::readCBFsIntoMemForIUS(NAString& sampleTableName,
group->delayedRead = FALSE;
NAString cbfFile(cbfFilePrefix);
- str_itoa(group->colSet[0].colnum, buffer);
- cbfFile.append(buffer);
+ cbfFile.append(group->cbfFileNameSuffix());
if (stat(cbfFile, &sts) == 0) {
if ( bufSz < sts.st_size ) {
@@ -6672,10 +6944,9 @@ Int32 HSGlobalsClass::writeCBFstoDiskForIUS(NAString& sampleTableName,
HSColGroupStruct* group
)
{
- // We save the CBFs as binary files on disk. One CBF maps to one binary file.
- // The path of the directory for these files is specified in CQD
- // USTAT_IUS_PERSISTENT_CBF_PATH, and the cbf binary file name is
- // sampleTableName + '.' + 'col_position_in_table';
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: write CBF files to disk");
NAString path =
ActiveSchemaDB()->getDefaults().getValue(USTAT_IUS_PERSISTENT_CBF_PATH);
@@ -6688,21 +6959,21 @@ Int32 HSGlobalsClass::writeCBFstoDiskForIUS(NAString& sampleTableName,
UInt64 totalSpaceInBlocks = 0;
- if ( !getTotalDiskSizeInBlocks(path, totalSpaceInBlocks) )
+ if ( !getTotalDiskSizeInBlocks(path, totalSpaceInBlocks) ) {
+ if (LM->LogNeeded())
+ LM->StopTimer();
return 0;
+ }
UInt64 totalAllowedInBlocks = MINOF(totalCBFsizeInMB * 1024 / 2,
totalSpaceInBlocks * percentage);
- NAString cbfFilePrefix(path);
- cbfFilePrefix.append("/");
- cbfFilePrefix.append(sampleTableName);
- cbfFilePrefix.append(".");
+ NAString cbfFilePrefix;
+ getCBFFilePrefix(sampleTableName, cbfFilePrefix);
Lng32 sz;
Lng32 bufSz = 0;
char* bufptr = NULL;
- char buffer[20];
Int32 count = 0;
@@ -6711,9 +6982,7 @@ Int32 HSGlobalsClass::writeCBFstoDiskForIUS(NAString& sampleTableName,
if ( group->cbf && group->state == PROCESSED ) {
NAString cbfFile(cbfFilePrefix);
- str_itoa(group->colSet[0].colnum, buffer);
- cbfFile.append(buffer);
-
+ cbfFile.append(group->cbfFileNameSuffix());
Lng32 cbfSz = group->cbf->getTotalMemSize();
@@ -6736,8 +7005,8 @@ Int32 HSGlobalsClass::writeCBFstoDiskForIUS(NAString& sampleTableName,
char* buffer = bufptr;
sz = group->cbf->packIntoBuffer(buffer, FALSE /* no bytes swapping */ );
- assert( sz <= bufSz);
- assert( sz <= buffer - bufptr);
+ HS_ASSERT( sz <= bufSz);
+ HS_ASSERT( sz <= buffer - bufptr);
ssize_t wsz = write(fd, bufptr, sz);
@@ -6750,42 +7019,39 @@ Int32 HSGlobalsClass::writeCBFstoDiskForIUS(NAString& sampleTableName,
}
+ // Make sure we don't write it again on next batch.
+ delete group->cbf;
+ group->cbf = NULL;
}
group = group->next;
}
NADELETEBASIC(bufptr, STMTHEAP);
+ if (LM->LogNeeded())
+ LM->StopTimer();
return count;
}
Int32 HSGlobalsClass::deletePersistentCBFsForIUS(NAString& sampleTableName,
- HSColGroupStruct* group)
+ HSColGroupStruct* group,
+ SortState stateToDelete)
{
- // Before we have the PERSISTENT_DATA table available to us, we will
- // save the CBFs as binary files on disk. One CBF maps to one binary file.
- // The path of the directory for these files is specified in CQD
- // USTAT_IUS_PERSISTENT_CBF_PATH, and the cbf binary file name is
- // sampleTableName + '.' + 'colName';
-
- NAString path =
- ActiveSchemaDB()->getDefaults().getValue(USTAT_IUS_PERSISTENT_CBF_PATH);
-
- NAString cbfFilePrefix(path);
- cbfFilePrefix.append("/");
- cbfFilePrefix.append(sampleTableName);
- cbfFilePrefix.append(".");
- char buffer[20];
+ NAString cbfFilePrefix;
+ getCBFFilePrefix(sampleTableName, cbfFilePrefix);
while (group) {
- if ( group->cbf && group->state == PENDING ) {
+ if ( group->cbf && group->state == stateToDelete ) {
NAString cbfFile(cbfFilePrefix);
- str_itoa(group->colSet[0].colnum, buffer);
- cbfFile.append(buffer);
+ cbfFile.append(group->cbfFileNameSuffix());
remove(cbfFile.data());
+
+ // Make sure this unused CBF does not get persisted.
+ delete group->cbf;
+ group->cbf = NULL;
}
group = group->next;
@@ -6797,8 +7063,13 @@ Int32 HSGlobalsClass::deletePersistentCBFsForIUS(NAString& sampleTableName,
Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABoolean& ranOut, Int32& colsSelected)
{
HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: selectIUSBatch()");
colsSelected = 0;
+ Int32 colsSuggested = 0; // number of cols we try to allocate
+ iusSampleDeletedInMem->depopulate();
+ iusSampleInsertedInMem->depopulate();
Int64 memAllowed = getMaxMemory();
Int64 memLeft = memAllowed;
@@ -6806,6 +7077,8 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
Lng32 retcode = 0;
Int64 tableUID = objDef->getObjectUID();
+ char UIDStr[30];
+ convertInt64ToAscii(tableUID,UIDStr);
Lng32 colnum = 0;
UInt32 histID = 0;
@@ -6814,11 +7087,20 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
Int64 totalUEC = 0;
Int64 v2;
- HSCliStatement histCursor(HSCliStatement::CURSOR107_MX_2300,
- (char*)hstogram_table->data(),
- (char*)&tableUID,
- (char*)&colnum);
+ // SELECT HISTOGRAM_ID, INTERVAL_COUNT, ROWCOUNT, TOTAL_UEC, V2
+ // FROM SB_HISTOGRAMS
+ // WHERE TABLE_UID = tableUID AND COLCOUNT = 1 AND COLUMN_NUMBER = CAST(? AS INTEGER)
+
+ HSErrorCatcher errorCatcher(retcode, - UERR_INTERNAL_ERROR, "HSGlobalsClass::selectIUSBatch", TRUE);
+ NAString query = "SELECT HISTOGRAM_ID, INTERVAL_COUNT, ROWCOUNT, TOTAL_UEC, V2 FROM ";
+ query += *hstogram_table;
+ query += " WHERE TABLE_UID = ";
+ query += UIDStr;
+ query += " AND COLCOUNT = 1 AND COLUMN_NUMBER = CAST(? AS INTEGER)"; // single column histograms only
+
+ HSCursor histCursor;
+ histCursor.prepareQuery(query.data(), 1, 5); // 1 input parameter, 5 output
// Memory required by RUS has been estimated in prepareForIUS().
// Here we need to add the extra amount needed by IUS. Do it for each group below.
@@ -6853,7 +7135,7 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
}
colnum = group->colSet[0].colnum;
- retcode = histCursor.open();
+ retcode = histCursor.open(1, (void*)&colnum);
HSHandleError(retcode);
retcode = histCursor.fetch(5, (void*)&histID, (void*)&intvlCount,
(void*)&totalRowCount, (void*)&totalUEC,
@@ -6927,15 +7209,17 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
insGroup->state = PENDING;
- colsSelected++;
+ colsSuggested++;
memLeft -= totMemNeeded;
} else {
- // the memory is not enough for the group. Do nothing.
- // The group is still in UNPROCESSED state.
+ // Not enough memory for the group. Leave the group in UNPROCESSED state,
+ // and get rid of the HSHistogram object we created for it.
ranOut = TRUE;
+ delete group->groupHist;
+ group->groupHist = NULL;
if (LM->LogNeeded()) {
- sprintf(LM->msg, "Not enough memory: memLeft=" PF64 "totMemNeeded=", memLeft);
+ sprintf(LM->msg, "Not enough memory for %s: memLeft=" PF64 " totMemNeeded=", group->colNames->data(), memLeft);
formatFixedNumeric((Int64)totMemNeeded, 0, LM->msg+strlen(LM->msg));
LM->Log(LM->msg);
sprintf(LM->msg, "group->memNeeded="PF64"", group->memNeeded);
@@ -6983,15 +7267,13 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
insGroup = insGroup->next;
}
- // Now allocate memory for singleGroup, inMemDelete and inMemInsert table,
- // for each column in PENDING state.
- allocateMemoryForColumns(singleGroup, futureRows, NULL);
-
- allocateMemoryForColumns(iusSampleDeletedInMem->getColumns(),
- iusSampleDeletedInMem->getNumRows(), NULL);
-
- allocateMemoryForColumns(iusSampleInsertedInMem->getColumns(),
- iusSampleInsertedInMem->getNumRows(), NULL);
+ // Now allocate memory for singleGroup, inMemDelete and inMemInsert table,
+ // for each column in PENDING state.
+ colsSelected = allocateMemoryForIUSColumns(singleGroup, futureRows,
+ iusSampleDeletedInMem->getColumns(),
+ iusSampleDeletedInMem->getNumRows(),
+ iusSampleInsertedInMem->getColumns(),
+ iusSampleInsertedInMem->getNumRows());
if (LM->LogNeeded())
{
@@ -7008,8 +7290,10 @@ Lng32 HSGlobalsClass::selectIUSBatch(Int64 currentRows, Int64 futureRows, NABool
}
group = group->next;
}
- sprintf(LM->msg, "return from selectIUSBatch(): count=%d", colsSelected);
+ sprintf(LM->msg, "return from selectIUSBatch(): columns originally selected = %d, "
+ "columns able to allocate = %d", colsSuggested, colsSelected);
LM->Log(LM->msg);
+ LM->StopTimer();
}
return retcode;
@@ -7040,8 +7324,11 @@ Lng32 HSGlobalsClass::incrementHistograms()
} else {
if ( retcode == 0 ) {
group->state = PROCESSED; // IUS successful.
- delGroup->state = PROCESSED; // IUS successful.
- insGroup->state = PROCESSED; // IUS successful.
+ group->freeISMemory();
+ delGroup->state = PROCESSED; // IUS successful.
+ delGroup->freeISMemory();
+ insGroup->state = PROCESSED; // IUS successful.
+ insGroup->freeISMemory();
} else
HSHandleError(retcode);
}
@@ -7147,9 +7434,14 @@ Lng32 HSGlobalsClass::initIUSIntervals(HSColGroupStruct* smplGroup,
UInt32 histID,
Int16 numIntervals)
{
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ LM->StartTimer("IUS: initIUSIntervals()");
typedef Int16 LenType;
Lng32 retcode = 0;
Int64 tableUID = objDef->getObjectUID();
+ char UIDStr[30];
+ convertInt64ToAscii(tableUID,UIDStr);
Int64 rowCount;
Int16 intvlNum;
@@ -7160,10 +7452,26 @@ Lng32 HSGlobalsClass::initIUSIntervals(HSColGroupStruct* smplGroup,
NAWchar boundarySpec[HS_MAX_UCS_BOUNDARY_CHAR + 1]; // +1 for 2-byte count
NAWchar MFV[HS_MAX_UCS_BOUNDARY_CHAR + 1];
- HSCliStatement intvlCursor(HSCliStatement::SHOWINT_MX_2300,
- (char*)hsintval_table->data(),
- (char*)&tableUID,
- (char*)&histID);
+ char histIDStr[20];
+ sprintf(histIDStr,"%u",histID);
+
+ // SELECT several columns
+ // FROM SB_HISTOGRAM_INTERVALS
+ // WHERE TABLE_UID = tableUID AND HISTOGRAM_ID = histID
+ // ORDER BY INTERVAL_NUMBER
+
+ NAString query = "SELECT INTERVAL_NUMBER, INTERVAL_ROWCOUNT, INTERVAL_UEC,"
+ " INTERVAL_BOUNDARY, STD_DEV_OF_FREQ, V1, V2, V5 FROM ";
+ query += *hsintval_table;
+ query += " WHERE TABLE_UID = ";
+ query += UIDStr;
+ query += " AND HISTOGRAM_ID = ";
+ query += histIDStr;
+ query += " ORDER BY INTERVAL_NUMBER";
+
+ HSCursor intvlCursor(STMTHEAP,HS_INTERVAL_STMT_ID);
+ intvlCursor.prepareQuery(query.data(), 0, 8); // no input parameters, 8 output
+
retcode = intvlCursor.open();
HSHandleError(retcode);
@@ -7172,8 +7480,8 @@ Lng32 HSGlobalsClass::initIUSIntervals(HSColGroupStruct* smplGroup,
while (TRUE)
{
- retcode = intvlCursor.fetch(8, (void*)&rowCount,
- (void*)&intvlNum,
+ retcode = intvlCursor.fetch(8, (void*)&intvlNum,
+ (void*)&rowCount,
(void*)&uec,
(void*)boundarySpec,
(void*)&stddev, (void*)&v1, (void*)&v2,
@@ -7237,6 +7545,8 @@ Lng32 HSGlobalsClass::initIUSIntervals(HSColGroupStruct* smplGroup,
}
+ if (LM->LogNeeded())
+ LM->StopTimer();
return retcode;
}
@@ -8076,6 +8386,35 @@ NABoolean HSGlobalsClass::allMCGroupsProcessed(NABoolean forIS)
}
+// This function is called by the HS_ASSERT macro to take care of some things
+// before triggering an assertion failure:
+// - Log the assertion failure if logging is enabled.
+// - Roll back transaction if one is in progress.
+// - Put an assertion error in the diagnostics area. This is supposed to be
+// done by code executed due to the macro HS_ASSERT invokes for the assertion
+// failure, but it does not always work properly. Doing it here prevents it
+// from being attempted downstream.
+// The parameters are the text of the assertion, and the file and line at which
+// it occurred.
+void HSGlobalsClass::preAssertionFailure(const char* condition,
+ const char* fileName,
+ Lng32 lineNum)
+{
+ HSTranMan *TM = HSTranMan::Instance();
+ HSLogMan *LM = HSLogMan::Instance();
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "***[ERROR] INTERNAL ASSERTION (%s) AT %s:%i", condition, fileName, lineNum);
+ LM->Log(LM->msg);
+ }
+ if (TM->StartedTransaction())
+ TM->Rollback();
+ diagsArea << DgSqlCode(arkcmpErrorAssert)
+ << DgString0(condition)
+ << DgString1(fileName)
+ << DgInt0(lineNum);
+}
+
/****************************************************************/
/* METHOD: getRetcodeFromDiags() */
@@ -10589,10 +10928,27 @@ Int32 HSGlobalsClass::selectSortBatch(Int64 rows,
return count;
}
+// Reduce the percentage of physical memory to limit internal sort to, following
+// an allocation failure that proved our previous estimate too high. We arbitrarily
+// reduce it to 90% of what it was. We could get smarter about this, and base the
+// reduction on how much of what we recommended was successfully allocated before
+// the failure.
+void HSGlobalsClass::memReduceAllowance()
+{
+ HSLogMan *LM = HSLogMan::Instance();
+
+ ISMemPercentage_ *= .9f;
+
+ if (LM->LogNeeded())
+ {
+ sprintf(LM->msg, "Reducing ISMemPercentage_ to %f", ISMemPercentage_);
+ LM->Log(LM->msg);
+ }
+}
+
// Takes corrective action when a memory allocation for internal sort could not
// be made. Remove the offending column and remaining unallocated columns from
-// the current internal sort batch, and reduce the percentage of available memory
-// to request.
+// the current internal sort batch.
//
// Parameters:
// failedGroup - The single-col group which could not be allocated for.
@@ -10610,8 +10966,8 @@ void HSGlobalsClass::memRecover(HSColGroupStruct* failedGroup,
if (LM->LogNeeded())
{
LM->Log("<<<Recovering from failed memory allocation for internal sort");
- sprintf(LM->msg, "Memory allocation failed for %s",
- failedGroup->colSet[0].colname->data());
+ sprintf(LM->msg, "Memory allocation failed for %s (" PF64 " rows)",
+ failedGroup->colSet[0].colname->data(), rows);
LM->Log(LM->msg);
}
@@ -10683,19 +11039,127 @@ void HSGlobalsClass::memRecover(HSColGroupStruct* failedGroup,
}
while (grp = grp->next);
- // Reduce the percentage of physical memory to limit internal sort to, since
- // our previous estimate was obviously too high. We arbitrarily reduce it to
- // 90% of what it was. We could get smarter about this, and base the reduction
- // on how much of what we recommended was successfully allocated before the
- // failure.
- ISMemPercentage_ *= .9f;
+ if (LM->LogNeeded())
+ LM->Log(">>>Finished recovery from failed memory allocation for internal sort");
+}
+// The data read into memory for IUS consists not only of the primary data
+// (from the existing persistent sample), but also the data for the rows to
+// be removed from the sample, and those to be inserted into the sample. The
+// allocation of memory for these three sets of data for a column must be
+// kept consistent, such that on a given pass, the three data sets for a
+// column should either all be in memory, or none of them.
+//
+// This function attempts to allocate the needed memory for all three data
+// sets for a given column selected for an IUS batch. If an allocation failure
+// occurs for one of the data sets, it ensures that any prior allocation for
+// a corresponding data set is undone, and that the state of corresponding
+// groups are all the same (typically removing the column from the current
+// batch by changing its state from PENDING to UNPROCESSED).
+Int32 HSGlobalsClass::allocateMemoryForIUSColumns(HSColGroupStruct* group,
+ Int64 rows,
+ HSColGroupStruct* delGroup,
+ Int64 delRows,
+ HSColGroupStruct* insGroup,
+ Int64 insRows)
+{
+ HSLogMan *LM = HSLogMan::Instance();
if (LM->LogNeeded())
- {
- sprintf(LM->msg, "Reducing ISMemPercentage_ to %f", ISMemPercentage_);
- LM->Log(LM->msg);
- LM->Log(">>>Finished recovery from failed memory allocation for internal sort");
- }
+ LM->StartTimer("Allocate storage for IUS columns");
+
+ Int32 numCols = 0;
+ HSColGroupStruct* firstPendingGroup = NULL;
+
+ // To simplify the logic of keeping the three groups in sync, place them
+ // and their row counts in arrays. On each iteration the array elements
+ // are updated to point to the next group in the respective list.
+ HSColGroupStruct* groupArr[] = {group, delGroup, insGroup};
+ Int64 rowsArr[] = {rows, delRows, insRows};
+
+ NABoolean gotMemory = TRUE;
+
+ // Create storage for query results.
+ do
+ {
+ if (groupArr[0]->state != PENDING)
+ {
+ // Skip column not selected for this batch by advancing all 3 groups.
+ for (Int16 i=0; i<3; i++)
+ groupArr[i] = groupArr[i]->next;
+ continue;
+ }
+
+ if (!firstPendingGroup)
+ firstPendingGroup = groupArr[0];
+
+ // Allocate all memory needed for storing values of each group. If unable
+ // to do so for all groups, make necessary adjustments to group states and
+ // set flag indicating memory shortfall.
+ for (Int16 i=0; i<3 && gotMemory; i++)
+ {
+ if (!groupArr[i]->allocateISMemory(rowsArr[i]))
+ {
+ // Recover from failed allocation (free any partial allocation and
+ // reset the group's state to UNPROCESSED). Also do this for any
+ // groups already allocated, e.g., if the allocation fails for
+ // delGroup, back out the allocation for the primary group for the
+ // column in question.
+ Int16 j;
+ HSColGroupStruct *grpi, *grpj;
+ for (j=i; j>=0; j--)
+ {
+ memRecover(groupArr[j], groupArr[0] == firstPendingGroup, rowsArr[j], NULL);
+ if (j < i) // free memory for groups in this set
+ groupArr[j]->freeISMemory(); // that were already allocated
+ }
+
+ // For groups not allocated yet, don't need to free anything, but
+ // make sure the states of corresponding groups are the same.
+ // memRecover() will have changed the PENDING state of some of the
+ // columns, and if the corresponding delGroup and/or insGroup are
+ // not changed, they will be part of the queries to read the sample
+ // decrement/increment, for columns that are not being processed in
+ // this batch.
+ for (j=i+1; j<3; j++)
+ {
+ grpi = groupArr[i];
+ grpj = groupArr[j];
+ while (grpi)
+ {
+ grpj->state = grpi->state;
+ grpi = grpi->next;
+ grpj = grpj->next;
+ }
+ }
+ gotMemory = FALSE;
+ memReduceAllowance();
+ }
+ else
+ {
+ // Allocation was successful.
+ groupArr[i]->nextData = groupArr[i]->data;
+ groupArr[i]->mcis_nextData = groupArr[i]->mcis_data;
+ }
+ }
+
+ // If the allocation was successful, increment the count of columns for
+ // which memory was allocated, and advance to the next element in each
+ // sequence of groups (primary, delete, and insert). If the allocation
+ // was not successful, the loop will exit (recovery from the allocation
+ // failure has been performed within the loop).
+ if (gotMemory)
+ {
+ for (Int16 i=0; i<3; i++)
+ groupArr[i] = groupArr[i]->next;
+ numCols++;
+ }
+
+ } while (gotMemory && groupArr[0]);
+
+ if (LM->LogNeeded())
+ LM->StopTimer();
+
+ return numCols;
}
// Allocates memory needed for internal sort for all columns marked as PENDING.
@@ -10731,6 +11195,7 @@ Int32 HSGlobalsClass::allocateMemoryForColumns(HSColGroupStruct* group,
if (!group->allocateISMemory(rows))
{
memRecover(group, group == firstPendingGroup, rows, mgr);
+ memReduceAllowance();
break;
}
@@ -10863,8 +11328,9 @@ Lng32 HSGlobalsClass::readColumnsIntoMem(HSCursor *cursor, Int64 rows)
// processInternalSortNulls() to a separate new method in the future.
retcode = processInternalSortNulls(cursor->rowsetSize(), singleGroup);
HSHandleError(retcode);
- retcode = cursor->setRowsetPointers(singleGroup,
- (Lng32)MINOF(MAX_ROWSET, rowsLeft));
+ Lng32 rowsetSize = (Lng32)MINOF(MAX_ROWSET, rowsLeft);
+ if (rowsetSize > 0)
+ retcode = cursor->setRowsetPointers(singleGroup,rowsetSize);
}
}
if (retcode < 0) HSHandleError(retcode) else retcode=0; // Set to 0 for warnings.
@@ -11236,6 +11702,31 @@ Int64 placeWidePivot(T* sortArr, Int64 lowInx, Int64 highInx, Int64 pivotInx,
return (pivotWidth >= highInx - lowInx ? -1 : storePtr - sortArr);
}
+
+template <class T>
+void checkForBackwardness(HSGlobalsClass * hsGlobals, HSColGroupStruct *group, T * listitem1, T * listitem2)
+{
+ if (*listitem1 > *listitem2)
+ {
+ group->backwardWarningCount++;
+ if (group->backwardWarningCount < 5) // report this warning at most 5 times per column
+ {
+ // raise a warning that we found data in backwards order, which means
+ // we might get out-of-order histograms, which is bad
+ hsGlobals->diagsArea << DgSqlCode(UERR_UNEXPECTED_BACKWARDS_DATA)
+ << DgString0(group->colSet[0].colname->data());
+ }
+ }
+}
+
+template < >
+void checkForBackwardness(HSGlobalsClass * hsGlobals, HSColGroupStruct *group,
+MCWrapper * listitem1, MCWrapper * listitem2)
+{
+ // TODO: write this method when necessary; it's a no-op for now
+}
+
+
// The data in the column's data array has been sorted but not grouped.
// Iterate over the values, counting duplicates. When a new value is
// encountered, create a new group, consisting of a distinct value and
@@ -11295,6 +11786,7 @@ do
valueCountIndex++;
numRows=0;
}
+ checkForBackwardness(hsGlobals,group,listitem1,listitem2);
listitem1++;
listitem2++;
valueIndex++;
@@ -11628,6 +12120,60 @@ NAString& HSGlobalsClass::getWherePredicateForIUS()
return (*ius_where_condition_text);
}
+// Return the following string in the queryText parameter:
+// delete from <smplTable> where <whereCondition>
+void HSGlobalsClass::generateIUSDeleteQuery(const NAString& smplTable,
+ NAString& queryText)
+{
+ queryText = "DELETE FROM ";
+
+ queryText.append(smplTable.data());
+
+ NAString& whereClause = getWherePredicateForIUS();
+ if (whereClause.length() > 0) {
+ queryText.append(" WHERE ");
+ queryText.append(whereClause);
+ }
+}
+
+// Create statement to add rows to the IUS persistent sample table.
+// upsert using load into into <smplTable>...
+//
+// If doing full IUS, the new sample table rows are already in the temporary
+// _I table, and the source for the upsert is
+// (select * from <smplTable>_I)
+//
+// If a limited IUS (update persistent sample table and generate histograms),
+// the source for the upsert is the source table with the IUS where predicate
+// and sampling rate applied:
+// (select * from <sourceTable> where <predicate> sample random <sampleRate> percent)
+void HSGlobalsClass::generateIUSSelectInsertQuery(const NAString& smplTable,
+ const NAString& sourceTable,
+ NAString& queryText)
+{
+ queryText.append("UPSERT USING LOAD INTO "); // for algorithm 1
+ queryText.append(smplTable.data());
+ queryText.append(" (SELECT * FROM ");
+
+ if (CmpCommon::getDefault(USTAT_INCREMENTAL_UPDATE_STATISTICS) == DF_ON)
+ {
+ queryText.append(smplTable.data());
+ queryText.append("_I)");
+ }
+ else
+ {
+ queryText.append(sourceTable.data());
+ queryText.append(" where ");
+ queryText.append(getWherePredicateForIUS());
+ NAString sampleOpt;
+ createSampleOption(SAMPLE_RAND_1,
+ sampleRateAsPercetageForIUS * 100.0,
+ sampleOpt, 0, 0);
+ queryText.append(sampleOpt);
+ queryText.append(")");
+ }
<TRUNCATED>
[3/6] incubator-trafodion git commit: [TRAFODION-2140] Move IUS
feature to open source
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/seabase/EXPECTED002
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED002 b/core/sql/regress/seabase/EXPECTED002
index 83a8bb5..2888c5b 100644
--- a/core/sql/regress/seabase/EXPECTED002
+++ b/core/sql/regress/seabase/EXPECTED002
@@ -183,8 +183,9 @@ T002SCH.T002T2 1
T002SCH.T002T2 0 ReadRequestsCount: 1 WriteRequestsCount: 1
T002SCH.SB_HIS 0 ReadRequestsCount: 0 WriteRequestsCount: 0
T002SCH.T002T1 0 ReadRequestsCount: 2 WriteRequestsCount: 1
+T002SCH.SB_PER 0 ReadRequestsCount: 0 WriteRequestsCount: 0
---- 3 row(s) selected.
+--- 4 row(s) selected.
>>
>>select left(trim(schema_name) || '.' || trim(object_name), 14),
+> sum(store_file_size),
@@ -200,8 +201,9 @@ T002SCH.T002T1 0
T002SCH.T002T2 0 ReadRequestsCount: 1 WriteRequestsCount: 1
T002SCH.SB_HIS 0 ReadRequestsCount: 0 WriteRequestsCount: 0
T002SCH.T002T1 0 ReadRequestsCount: 2 WriteRequestsCount: 1
+T002SCH.SB_PER 0 ReadRequestsCount: 0 WriteRequestsCount: 0
---- 3 row(s) selected.
+--- 4 row(s) selected.
>>
>>get region stats for table t002t1;
@@ -446,7 +448,35 @@ Stats Details
=============
RegionNum: 1
- RegionName: TRAFODION.T002SCH.SB_HISTOGRAM_INTERVALS/94b6ffefc5df9a62180e5a2401e18d9e
+ RegionName: TRAFODION.T002SCH.SB_HISTOGRAM_INTERVALS/72bc6242dfa139b77bd15125cf46593f
+ NumStores: 2
+ NumStoreFiles: 0
+ UncompressedSize: 0 (less than 1MB)
+ StoreFileSize: 0 (less than 1MB)
+ MemStoreSize: 0 (less than 1MB)
+ ReadRequestsCount: 0
+ WriteRequestsCount: 0
+
+
+Stats Summary
+=============
+
+ ObjectName: TRAFODION.T002SCH.SB_PERSISTENT_SAMPLES
+ NumRegions: 1
+ RegionsLocation: /hbase/data/default
+ TotalNumStores: 2
+ TotalNumStoreFiles: 0
+ TotalUncompressedSize: 0
+ TotalStoreFileSize: 0
+ TotalMemStoreSize: 0
+ TotalReadRequestsCount: 0
+ TotalWriteRequestsCount: 0
+
+Stats Details
+=============
+
+ RegionNum: 1
+ RegionName: TRAFODION.T002SCH.SB_PERSISTENT_SAMPLES/121ea772487c23e7f165e29db4647fd4
NumStores: 2
NumStoreFiles: 0
UncompressedSize: 0 (less than 1MB)
@@ -733,6 +763,20 @@ Stats Summary
Stats Summary
=============
+ ObjectName: TRAFODION.T002SCH.SB_PERSISTENT_SAMPLES
+ NumRegions: 1
+ RegionsLocation: /hbase/data/default
+ TotalNumStores: 2
+ TotalNumStoreFiles: 0
+ TotalUncompressedSize: 0
+ TotalStoreFileSize: 0
+ TotalMemStoreSize: 0
+ TotalReadRequestsCount: 0
+ TotalWriteRequestsCount: 0
+
+Stats Summary
+=============
+
ObjectName: TRAFODION.T002SCH.T002T1
NumRegions: 1
RegionsLocation: /hbase/data/default
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/seabase/EXPECTED012
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED012 b/core/sql/regress/seabase/EXPECTED012
index 4db69b5..e43f13c 100644
--- a/core/sql/regress/seabase/EXPECTED012
+++ b/core/sql/regress/seabase/EXPECTED012
@@ -284,6 +284,7 @@ Tables in Schema TRAFODION.ZSCHEMA
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
+SB_PERSISTENT_SAMPLES
T012T11
T012T12
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/seabase/EXPECTED026
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED026 b/core/sql/regress/seabase/EXPECTED026
index d7b96bc..4151c08 100644
--- a/core/sql/regress/seabase/EXPECTED026
+++ b/core/sql/regress/seabase/EXPECTED026
@@ -221,7 +221,7 @@
>>delete from "_MD_".objects where catalog_name = 'TRAFODION' and
+> schema_name = 'SCH026' and object_type = 'PK';
---- 6 row(s) deleted.
+--- 7 row(s) deleted.
>>cqd hide_indexes reset;
--- SQL operation complete.
@@ -292,7 +292,7 @@ Metadata Cleanup: started, check only
End: Cleanup Orphan Hbase Entries (0 entries found)
Start: Cleanup Inconsistent Objects Entries
- End: Cleanup Inconsistent Objects Entries (6 entries found)
+ End: Cleanup Inconsistent Objects Entries (7 entries found)
Start: Cleanup Inconsistent Views Entries
End: Cleanup Inconsistent Views Entries (0 entries found)
@@ -327,7 +327,7 @@ Metadata Cleanup: started, check only
End: Cleanup Orphan Hbase Entries (1 entry found)
Start: Cleanup Inconsistent Objects Entries
- End: Cleanup Inconsistent Objects Entries (8 entries found)
+ End: Cleanup Inconsistent Objects Entries (9 entries found)
Start: Cleanup Inconsistent Views Entries
End: Cleanup Inconsistent Views Entries (0 entries found)
@@ -353,10 +353,11 @@ Metadata Cleanup: started, check only
Entry #3(OBJECT): TRAFODION.SCH026.T026T2_498588266_9589
Entry #4(OBJECT): TRAFODION.SCH026.SB_HISTOGRAMS_PK
Entry #5(OBJECT): TRAFODION.SCH026.SB_HISTOGRAM_INTERVALS_PK
- Entry #6(OBJECT): TRAFODION.SCH026.LOBMD_00958229992622_255194156_9672
- Entry #7(OBJECT): TRAFODION.SCH026.LOBDescChunks_009582_898194156_9672
- Entry #8(UID): 958229992622171473
- End: Cleanup Inconsistent Objects Entries (8 entries found)
+ Entry #6(OBJECT): TRAFODION.SCH026.SB_PERSISTENT_SAMPLES_PK
+ Entry #7(OBJECT): TRAFODION.SCH026.LOBMD_03901336551423_331332878_4561
+ Entry #8(OBJECT): TRAFODION.SCH026.LOBDescChunks_039013_645332878_4561
+ Entry #9(UID): 3901336551423203710
+ End: Cleanup Inconsistent Objects Entries (9 entries found)
Start: Cleanup Inconsistent Views Entries
End: Cleanup Inconsistent Views Entries (0 entries found)
@@ -382,10 +383,11 @@ Metadata Cleanup: started
Entry #3(OBJECT): TRAFODION.SCH026.T026T2_498588266_9589
Entry #4(OBJECT): TRAFODION.SCH026.SB_HISTOGRAMS_PK
Entry #5(OBJECT): TRAFODION.SCH026.SB_HISTOGRAM_INTERVALS_PK
- Entry #6(OBJECT): TRAFODION.SCH026.LOBMD_00958229992622_255194156_9672
- Entry #7(OBJECT): TRAFODION.SCH026.LOBDescChunks_009582_898194156_9672
- Entry #8(UID): 958229992622171473
- End: Cleanup Inconsistent Objects Entries (8 entries cleaned up)
+ Entry #6(OBJECT): TRAFODION.SCH026.SB_PERSISTENT_SAMPLES_PK
+ Entry #7(OBJECT): TRAFODION.SCH026.LOBMD_03901336551423_331332878_4561
+ Entry #8(OBJECT): TRAFODION.SCH026.LOBDescChunks_039013_645332878_4561
+ Entry #9(UID): 3901336551423203710
+ End: Cleanup Inconsistent Objects Entries (9 entries cleaned up)
Start: Cleanup Inconsistent Views Entries
End: Cleanup Inconsistent Views Entries (0 entries cleaned up)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/regress/tools/runregr_compGeneral.ksh
----------------------------------------------------------------------
diff --git a/core/sql/regress/tools/runregr_compGeneral.ksh b/core/sql/regress/tools/runregr_compGeneral.ksh
index e7e87d7..e100654 100755
--- a/core/sql/regress/tools/runregr_compGeneral.ksh
+++ b/core/sql/regress/tools/runregr_compGeneral.ksh
@@ -226,7 +226,7 @@ fi
# sbtestfiles contains the list of tests to be run in seabase mode
if [ "$seabase" -ne 0 ]; then
- sbtestfiles="TEST001 TEST004 TEST005 TEST006 TEST011 TEST012 TEST013 TEST015 TEST071 TEST042 TESTTOK TESTTOK2 TEST062"
+ sbtestfiles="TEST001 TEST004 TEST005 TEST006 TEST011 TEST012 TEST013 TEST015 TEST023 TEST071 TEST042 TEST043 TESTTOK TESTTOK2 TEST062"
sbprettyfiles=
for i in $prettyfiles; do
for j in $sbtestfiles; do
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
index 6079782..a2afca3 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
@@ -234,7 +234,9 @@ short CmpSeabaseMDcleanup::validateInputValues(
inObjType.data(),
outObjType, objectOwner_);
if ((objUID_ == -1) && // not found
- (inObjType != COM_UNKNOWN_OBJECT_LIT)) // type explicitly specified
+ (inObjType != COM_UNKNOWN_OBJECT_LIT &&
+ inObjType != COM_PRIVATE_SCHEMA_OBJECT_LIT &&
+ inObjType != COM_SHARED_SCHEMA_OBJECT_LIT)) // type explicitly specified
{
// check if there is another object type with the same name.
objUID_ = getCleanupObjectUID(cliInterface,
@@ -941,6 +943,7 @@ void CmpSeabaseMDcleanup::cleanupSchemaObjects(ExeCliInterface *cliInterface)
FALSE, FALSE
);
}
+ schObjList->advance();
}
// Now drop remaining objects
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp
index a3d834c..1a38b9c 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp
@@ -949,7 +949,9 @@ ComBoolean CmpSeabaseDDL::isHbase(const ComObjectName &name)
bool CmpSeabaseDDL::isHistogramTable(const NAString &name)
{
- if (name == HBASE_HIST_NAME || name == HBASE_HISTINT_NAME)
+ if (name == HBASE_HIST_NAME ||
+ name == HBASE_HISTINT_NAME ||
+ name == HBASE_PERS_SAMP_NAME )
return true;
return false;
@@ -6828,32 +6830,73 @@ short CmpSeabaseDDL::dropSeabaseStats(ExeCliInterface *cliInterface,
{
Lng32 cliRC = 0;
char buf[4000];
+
+ // delete any histogram statistics
str_sprintf(buf, "delete from %s.\"%s\".%s where table_uid = %Ld",
catName, schName, HBASE_HIST_NAME, tableUID);
cliRC = cliInterface->executeImmediate(buf);
- // if histogram table does not exist, return now without error
- // (could return on cliRC == 100, but that seems to happen
- // even when we deleted some histogram rows)
- if (cliRC == -4082)
- return 0;
- if (cliRC < 0)
+ // If the histogram table does not exist, don't bother checking
+ // the histogram intervals table.
+ //
+ // Note: cliRC == 100 happens when we delete some histogram rows
+ // and also when there are no histogram rows to delete, so we
+ // can't decide based on that.
+
+ if (cliRC != -4082)
{
- cliInterface->retrieveSQLDiagnostics(CmpCommon::diags());
- return -1;
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(CmpCommon::diags());
+ return -1;
+ }
+
+ str_sprintf(buf, "delete from %s.\"%s\".%s where table_uid = %Ld",
+ catName, schName, HBASE_HISTINT_NAME, tableUID);
+ cliRC = cliInterface->executeImmediate(buf);
+
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(CmpCommon::diags());
+ return -1;
+ }
}
- str_sprintf(buf, "delete from %s.\"%s\".%s where table_uid = %Ld",
- catName, schName, HBASE_HISTINT_NAME, tableUID);
- cliRC = cliInterface->executeImmediate(buf);
+ // Drop any persistent sample tables also. (It is possible that these
+ // might exist even if the histograms table does not exist.)
- if (cliRC < 0)
+ // Delete the row in the persistent_samples table if one exists
+
+ str_sprintf(buf, "select translate(sample_name using ucs2toutf8) from "
+ " (delete from %s.\"%s\".%s where table_uid = %Ld) as t",
+ catName, schName, HBASE_PERS_SAMP_NAME, tableUID);
+
+ Lng32 len = 0;
+ char sampleTableName[1000];
+
+ cliRC = cliInterface->executeImmediate(buf, (char*)&sampleTableName, &len, NULL);
+ if (cliRC == -4082) // if persistent_samples table does not exist
+ cliRC = 0; // then there isn't a persistent sample table
+ else if (cliRC < 0)
{
cliInterface->retrieveSQLDiagnostics(CmpCommon::diags());
return -1;
}
+ else if ((len > 0) && (sampleTableName[0])) // if we got a sample table name back
+ {
+ // try to drop the sample table
+ sampleTableName[len] = '\0';
+ str_sprintf(buf, "drop table %s", sampleTableName);
+ cliRC = cliInterface->executeImmediate(buf);
+ if (cliRC < 0)
+ {
+ cliInterface->retrieveSQLDiagnostics(CmpCommon::diags());
+ return -1;
+ }
+ }
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/sqlcomp/CmpSeabaseDDLmd.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLmd.h b/core/sql/sqlcomp/CmpSeabaseDDLmd.h
index e9d6a6f..add89e1 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLmd.h
+++ b/core/sql/sqlcomp/CmpSeabaseDDLmd.h
@@ -539,6 +539,27 @@ static const QString seabaseHistogramIntervalsDDL[] =
{" ; "}
};
+static const QString seabasePersistentSamplesDDL[] =
+{
+ {" create table "HBASE_PERS_SAMP_NAME" "},
+ {" ( "},
+ {" TABLE_UID LARGEINT NOT NULL NOT SERIALIZED, "},
+ {" REQUESTED_SAMPLE_ROWS LARGEINT NOT NULL NOT SERIALIZED, "},
+ {" ACTUAL_SAMPLE_ROWS LARGEINT NOT NULL NOT SERIALIZED, "},
+ {" SAMPLING_RATIO DOUBLE PRECISION NOT NULL NOT SERIALIZED, "},
+ {" CREATE_TIME TIMESTAMP(0) NOT NULL NOT SERIALIZED, "},
+ {" REASON CHAR(1) CHARACTER SET UCS2 NOT NULL NOT SERIALIZED, "},
+ {" SAMPLE_NAME VARCHAR(250) CHARACTER SET UCS2 NOT NULL NOT SERIALIZED, "},
+ {" LAST_WHERE_PREDICATE VARCHAR(250) CHARACTER SET UCS2 NOT NULL NOT SERIALIZED, "},
+ {" UPDATE_START_TIME TIMESTAMP(0) NOT NULL NOT SERIALIZED, "},
+ {" UPDATER_INFO VARCHAR(128) CHARACTER SET ISO88591 NOT NULL NOT SERIALIZED, "},
+ {" V1 VARCHAR(250) CHARACTER SET UCS2 NOT NULL NOT SERIALIZED, "},
+ {" V2 VARCHAR(250) CHARACTER SET UCS2 NOT NULL NOT SERIALIZED, "},
+ {" constraint "HBASE_PERS_SAMP_PK" primary key (TABLE_UID) "},
+ {" ) "},
+ {" ; "}
+};
+
struct MDTableInfo
{
// name of the new MD table.
@@ -646,6 +667,10 @@ static const MDTableInfo allMDHistInfo[] = {
{HBASE_HISTINT_NAME,
seabaseHistogramIntervalsDDL, sizeof(seabaseHistogramIntervalsDDL),
+ NULL, 0, FALSE},
+
+ {HBASE_PERS_SAMP_NAME,
+ seabasePersistentSamplesDDL, sizeof(seabasePersistentSamplesDDL),
NULL, 0, FALSE}
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index 988d382..1d825dc 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -3661,13 +3661,17 @@ short CmpSeabaseDDL::dropSeabaseTable2(
}
// drop SB_HISTOGRAMS and SB_HISTOGRAM_INTERVALS entries, if any
- // if the table that we are dropping itself is not a SB_HISTOGRAMS or SB_HISTOGRAM_INTERVALS table
+ // if the table that we are dropping itself is not a SB_HISTOGRAMS
+ // or SB_HISTOGRAM_INTERVALS table or sampling tables
// TBD: need to change once we start updating statistics for external
// tables
if (! (tableName.isExternalHive() || tableName.isExternalHbase()) )
{
if (objectNamePart != "SB_HISTOGRAMS" &&
- objectNamePart != "SB_HISTOGRAM_INTERVALS")
+ objectNamePart != "SB_HISTOGRAM_INTERVALS" &&
+ objectNamePart != "SB_PERSISTENT_SAMPLES" &&
+ strncmp(objectNamePart.data(),"TRAF_SAMPLE_",sizeof("TRAF_SAMPLE_")) != 0)
+
{
if (dropSeabaseStats(cliInterface,
catalogNamePart.data(),
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_auto.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_auto.h b/core/sql/ustat/hs_auto.h
index 3214545..5e642f6 100644
--- a/core/sql/ustat/hs_auto.h
+++ b/core/sql/ustat/hs_auto.h
@@ -37,6 +37,8 @@
*****************************************************************************
*/
+#include "hs_log.h"
+
#define NANewArray NewArray
// -----------------------------------------------------------------------
@@ -55,7 +57,7 @@ public:
: ptr_(source)
{
source = new(STMTHEAP) T[len];
- assert(source != NULL);
+ HS_ASSERT(source != NULL);
}
~NewArray()
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_cli.cpp b/core/sql/ustat/hs_cli.cpp
index 9e19a63..cc16591 100644
--- a/core/sql/ustat/hs_cli.cpp
+++ b/core/sql/ustat/hs_cli.cpp
@@ -122,28 +122,29 @@ private:
// doPrintPlan = if true, the plan for the query will be
// printed after the statement is prepared but before
// execution; if false we do a single CLI ExecDirect call
+// checkForMdam = if true, do a query on the Explain virtual table
+// to see if MDAM was used in the query being executed, and
+// display the result in the ulog.
// -----------------------------------------------------------------------
Lng32 HSExecDirect( SQLSTMT_ID * stmt
, SQLDESC_ID * srcDesc
, NABoolean doPrintPlan
+ , NABoolean checkForMdam
)
{
Lng32 retcode = 0;
- if (doPrintPlan)
+ HSLogMan *LM = HSLogMan::Instance();
+ if ((doPrintPlan || checkForMdam) && LM->LogNeeded())
{
retcode = SQL_EXEC_Prepare(stmt, srcDesc);
if (retcode >= 0) // ignore warnings
{
if (doPrintPlan)
- {
- HSLogMan *LM = HSLogMan::Instance();
- if (LM->LogNeeded())
- {
- printPlan(stmt);
- }
- }
+ printPlan(stmt);
+ if (checkForMdam)
+ checkMdam(stmt);
retcode = SQL_EXEC_ExecFetch(stmt,0,0);
}
}
@@ -173,6 +174,8 @@ Lng32 HSExecDirect( SQLSTMT_ID * stmt
// that should not disrupt execution, such as "schema already
// exists" when executing a Create Schema statement. 0 indicates
// there is no such expected error.
+// checkMdam = if TRUE, determine whether the query uses MDAM, and
+// include this information in the ulog.
// -----------------------------------------------------------------------
Lng32 HSFuncExecQuery( const char *dml
, short sqlcode
@@ -182,6 +185,7 @@ Lng32 HSFuncExecQuery( const char *dml
, const HSTableDef *tabDef
, NABoolean doRetry
, short errorToIgnore
+ , NABoolean checkMdam
)
{
HSLogMan *LM = HSLogMan::Instance();
@@ -272,7 +276,7 @@ Lng32 HSFuncExecQuery( const char *dml
if (!doRetry)
{
// execute immediate this statement
- retcode = HSExecDirect(&stmt, &srcDesc, srcTabRowCount != 0);
+ retcode = HSExecDirect(&stmt, &srcDesc, srcTabRowCount != 0, checkMdam);
// If retcode is > 0 or sqlcode is HS_WARNING, then set to 0 (no error/ignore).
if (retcode >= 0) retcode = 0;
// If sqlcode is HS_WARNING, then this means failures should be returned as
@@ -303,7 +307,7 @@ Lng32 HSFuncExecQuery( const char *dml
}
// execute immediate this statement
- retcode = HSExecDirect(&stmt, &srcDesc, srcTabRowCount != 0);
+ retcode = HSExecDirect(&stmt, &srcDesc, srcTabRowCount != 0, checkMdam);
// filter retcode for HSHandleError
HSFilterWarning(retcode);
@@ -431,7 +435,7 @@ Lng32 CreateHistTables (const HSGlobalsClass* hsGlobal)
// do NOT check volatile tables
if (hsGlobal->objDef->isVolatile()) return retcode;
- LM->StartTimer("Table creation");
+ LM->StartTimer("Create histogram tables");
NAString tableNotCreated;
// Call createHistogramTables to create any table that does not yet exist.
@@ -452,6 +456,25 @@ Lng32 CreateHistTables (const HSGlobalsClass* hsGlobal)
return retcode;
}
+Lng32 CreateSeabasePersSamples(const HSGlobalsClass* hsGlobal)
+ {
+ HSLogMan *LM = HSLogMan::Instance();
+ Lng32 retcode = 0;
+ ComObjectName tableName(hsGlobal->hsperssamp_table->data());
+ HSSqTableDef sampleDef(tableName, ANSI_TABLE);
+ if (!sampleDef.objExists()) //DROP existing sample table
+ {
+ if (LM->LogNeeded())
+ {
+ snprintf(LM->msg, sizeof(LM->msg), "Creating %s table for schema %s on demand.",
+ HBASE_PERS_SAMP_NAME, hsGlobal->catSch->data());
+ LM->Log(LM->msg);
+ }
+
+ retcode = CreateHistTables(hsGlobal);
+ }
+ return retcode;
+}
/***********************************************/
/* METHOD: HSSample create() member function */
@@ -461,27 +484,11 @@ Lng32 CreateHistTables (const HSGlobalsClass* hsGlobal)
/* RETCODE: 0 - successful */
/* non-zero otherwise */
/***********************************************/
-Lng32 HSSample::create(NABoolean unpartitioned, NABoolean isPersSample,
- NABoolean createDandI)
+Lng32 HSSample::create(NABoolean unpartitioned, NABoolean isPersSample)
{
- makeTableName(isPersSample); // assigns 'sampleTable' name for MX/MP.
- Lng32 retcode = 0;
- retcode = create(sampleTable, unpartitioned, isPersSample);
-
- if ( !retcode && createDandI ) {
- NAString tableD(sampleTable);
- tableD.append("_D");
-
- //retcode = create(tableD, unpartitioned, isPersSample);
- //HSHandleError(retcode);
-
- // The temp. table for sample I is created in ::generateSampleI() on demand.
- // Otherwise, we end up with 8102 error (rows duplicated) and very slow
- // performance.
- //NAString tableI(sampleTable);
- //tableI.append("_I");
- //retcode = create(tableI, unpartitioned, isPersSample);
- }
+ makeTableName(isPersSample); // assigns 'sampleTable' name for MX/MP.
+ Lng32 retcode = 0;
+ retcode = create(sampleTable, unpartitioned, isPersSample);
return retcode;
}
@@ -524,12 +531,14 @@ Lng32 HSSample::create(NAString& tblName, NABoolean unpartitioned, NABoolean isP
tableOptions = " WITH PARTITIONS";
// If a transaction is running, the table needs to be created as audited.
// Otherwise, create table as non-audited.
+ /* TEMPTEMP. Dave need to validate this change.
if (TM->InTransaction())
tableOptions += " ATTRIBUTE AUDIT";
else
tableOptions += " ATTRIBUTE NO AUDIT";
tableOptions += getTempTablePartitionInfo(unpartitioned, isPersSample);
+ */
ddl = "CREATE TABLE ";
ddl += tempTabName;
@@ -795,13 +804,6 @@ Lng32 HSSample::dropSample(NAString& sampTblName, HSTableDef *sourceTblDef)
LM->Log("\tDROP SAMPLE TABLE");
tempTabName = sampTblName;
- // LCOV_EXCL_START :nsk
- if (sourceTblDef->getObjectFormat() == SQLMP)
- {
- ComMPLoc tempObj(tempTabName, ComMPLoc::FILE);
- tempTabName = tempObj.getMPName();
- }
- // LCOV_EXCL_STOP
// If the table is not volatile, make the sample table 'droppable', except
// in the case of Trafodion, which does not support this Alter statement.
@@ -1100,7 +1102,7 @@ Lng32 HSTranMan::Rollback()
if (extTrans_)
LM->Log("ROBACKWORK(external transaction being used)");
else
- LM->Log("ROBACKWORK(no running traction)");
+ LM->Log("ROBACKWORK(no running transaction)");
}
}
return retcode_;
@@ -1239,84 +1241,96 @@ Lng32 HSTranController::lockTable(const char* tableName, NABoolean exclusive)
/* one instance of this class. */
/*****************************************************************************/
THREAD_P HSPersSamples* HSPersSamples::instance_ = 0;
-THREAD_P NAList<NAString>* HSPersSamples::persSampleTablesList_ = NULL;
-THREAD_P NAString *HSPersSamples::catalog_ = NULL;
-THREAD_P NAString *HSPersSamples::schema_ = NULL;
-HSPersSamples::HSPersSamples()
+
+HSPersSamples::HSPersSamples(const NAString & catalog, const NAString & schema)
+: catalog_(new (CTXTHEAP) NAString(catalog)),
+ schema_(new (CTXTHEAP) NAString(schema)),
+ triedCreatingSBPersistentSamples_(false)
{}
-/***********************************************/
-/* METHOD: Instance() */
-/* PURPOSE: Returns the instance of the class */
-/* and searches for or creates */
-/* PERSISTENT_SAMPLES table (list). */
-/* For every catalog for which a */
-/* PERSISTENT_SAMPLES table is known */
-/* to exist, the catalog name is added*/
-/* to the 'persSampleTablesList'. */
-/* RETCODE: 0 if no errors. */
-/* non-zero otherwise. */
-/* INPUT: 'catalog', the catalog of the table*/
-/* for which a sample is created. */
-/* 'createTable', if TRUE, create the */
-/* PERSISTENT_SAMPLES table for this */
-/* catalog if it doesn't exist. */
-/* NOTES: We need to instantiate using the */
-/* contextHeap because we need this */
-/* class to stay around through */
-/* multiple statements. */
-/* */
-/***********************************************/
+
+HSPersSamples::~HSPersSamples()
+ {
+ delete catalog_;
+ delete schema_;
+ }
+
+void HSPersSamples::setCatalogSchema(const NAString &catalog,
+ const NAString &schema)
+ {
+ if ((schema != *schema_) || (catalog != *catalog_))
+ {
+ *catalog_ = catalog;
+ *schema_ = schema;
+ triedCreatingSBPersistentSamples_ = false; // will try again on a new schema
+ }
+ }
+
+//
+// METHOD: Instance()
+// PURPOSE: Returns the instance of the class */
+// and searches for or creates */
+// SB_PERSISTENT_SAMPLES table (list). */
+// For every catalog for which a */
+// SB_PERSISTENT_SAMPLES table is known */
+// to exist, the catalog name is added*/
+// to the 'persSampleTablesList'. */
+// RETCODE: 0 if no errors. */
+// non-zero otherwise. */
+// INPUT: 'catalog', the catalog of the table*/
+// for which a sample is created. */
+// 'createTable', if TRUE, create the */
+// SB_PERSISTENT_SAMPLES table for this */
+// catalog if it doesn't exist. */
+// NOTES: We need to instantiate using the */
+// contextHeap because we need this */
+// class to stay around through */
+// multiple statements.
+//
HSPersSamples* HSPersSamples::Instance(const NAString &catalog,
- NABoolean createTable)
+ const NAString &schema)
{
Lng32 retcode = 0;
- if (instance_ == 0)
- instance_ = new (CTXTHEAP) HSPersSamples;
+ HSGlobalsClass *hs_globals = GetHSContext();
- if (persSampleTablesList_ == 0)
- persSampleTablesList_ = new(CTXTHEAP) NAList<NAString>;
- if (catalog_ == 0)
- catalog_ = new (CTXTHEAP) NAString("");
- if (schema_ == 0)
- schema_ = new (CTXTHEAP) NAString("");
+ if (instance_ == 0)
+ instance_ = new (CTXTHEAP) HSPersSamples(catalog,schema);
+ else
+ {
+ instance_->setCatalogSchema(catalog,schema);
+ }
- // Search for an instance of catalog in memory resident list.
if (HSGlobalsClass::isHiveCat(catalog))
{
- *catalog_ = HIVE_STATS_CATALOG;
- *schema_ = HIVE_STATS_SCHEMA;
+ NAString hiveStatsCatalog(HIVE_STATS_CATALOG);
+ NAString hiveStatsSchema(HIVE_STATS_SCHEMA_NO_QUOTES);
+ instance_->setCatalogSchema(hiveStatsCatalog,hiveStatsSchema);
}
- else if (HSGlobalsClass::isHbaseCat(catalog))
+ else if (HSGlobalsClass::isNativeHbaseCat(catalog))
{
- NAString sbCat = ActiveSchemaDB()->getDefaults().getValue(SEABASE_CATALOG);
-
- // *catalog_ = SEABASE_SYSTEM_CATALOG;
- *catalog_ = sbCat;
- *schema_ = SEABASE_SYSTEM_SCHEMA;
+ NAString hbaseStatsCatalog(HBASE_STATS_CATALOG);
+ NAString hbaseStatsSchema(HBASE_STATS_SCHEMA_NO_QUOTES);
+ instance_->setCatalogSchema(hbaseStatsCatalog,hbaseStatsSchema);
}
- else
+
+ // Create the SB_PERSISTENT_SAMPLES table if it does not already exist
+ // Do this at most once
+
+ if ((instance_) && (!instance_->triedCreatingSBPersistentSamples_))
{
- *catalog_ = catalog;
- *schema_ = "PUBLIC_ACCESS_SCHEMA";
+ retcode = CreateSeabasePersSamples(hs_globals);
+ instance_->triedCreatingSBPersistentSamples_ = true;
}
- if (!(*persSampleTablesList_).contains(*catalog_))
- {
- *(CmpCommon::diags()) << DgSqlCode(-4222)
- << DgString0("Persistent Sample Table");
- return 0;
- }
if (retcode) return 0;
else return instance_;
}
+
/***********************************************/
/* METHOD: find() */
-/* PURPOSE: finds a persistent sample table for*/
-/* UID and sample size. If a sample */
-/* is found that is 'obsolete', then */
-/* it will be removed and another will*/
-/* be searched for. If a persistent */
+/* PURPOSE: finds a manual (i.e., not IUS) */
+/* persistent sample table for UID */
+/* and sample size. If a persistent */
/* sample is found, the name will be */
/* returned in 'table', or blank if */
/* not. */
@@ -1335,63 +1349,69 @@ HSPersSamples* HSPersSamples::Instance(const NAString &catalog,
Lng32 HSPersSamples::find(HSTableDef *objDef, Int64 &actualRows, NABoolean isEstimate,
Int64 &sampleRows, double allowedDiff, NAString &table)
{
+ // Note: Earlier versions of this code supported multiple sample tables
+ // (with different sample ratios) for the same base table, which seems
+ // like overkill. So, in making this code work for Trafodion, we've made
+ // a simplifying assumption: Assume just one sample table per base table.
+ // (Indeed, we defined SB_PERSISTENT_SAMPLES to be keyed solely on the
+ // base table UID.)
+ //
+ // So, this method now ignores the actualRows, isEstimate, sampleRows and
+ // allowedDiff parameters. We can take them out later if this design
+ // decision becomes permanent.
+
Lng32 retcode = 0;
NABoolean removedObsolete=FALSE;
- table="";
+ table=""; // in case no sample table is found
NAWchar tempTabNameUCS2[408]; // first double byte is varchar length + a few extras NAWchars
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
NAString fromTable = persSampTblObjName.getExternalName();
- double percent;
- HSCliStatement::statementIndex del_stmt;
- HSCliStatement::statementIndex sel_stmt;
-
- // Create query to search for a matching persistent sample in
- // PERSISTENT_SAMPLES table. Persistent samples are matched
- // by the requested sample rowcount, not the actual that was
- // created, since the actual can vary due to VARCHARS, ...
- // A persistent sample is matching if for the same source table
- // and ABS(RQSTD_NUMROWS - 'sampleRows') <=
- // ('allowedDiff' * 'sampleRows'). The query will return the
- // persistent sample table name, sample row count, and sample %.
- sel_stmt = HSCliStatement::CURSOR_PST;
- del_stmt = HSCliStatement::DELETE_PST;
Int64 uid = objDef->getObjectUID();
- HSCliStatement cursorPST(sel_stmt,
- (char *)fromTable.data(),
- (char *)&uid,
- (char *)&sampleRows,
- (char *)&allowedDiff
- );
- retcode = cursorPST.open();
- HSHandleError(retcode);
+ char uidStr[30];
+ convertInt64ToAscii(uid,uidStr);
- do
- {
- // sel_stmt.fetch() will set 'tempTabName' to a persistent table name,
- // 'sampleRows' to the size, and 'percent' to the % of actual table if
- // one is found. Otherwise 'table' is set to "" and 'sampleRows' is not set.
- retcode = cursorPST.fetch (3, (void *)&tempTabNameUCS2[0], (void *)&sampleRows,
- (void *)&percent);
- // Handle errors from fetch. If retcode is 100 (meaning no more data),
- // then set retcode = 0.
- if (retcode)
+ HSCursor findSampleTableCursor1(STMTHEAP,"findSampleTable1");
+
+ NAString query = "SELECT SAMPLE_NAME FROM ";
+ query += fromTable;
+ query += " WHERE TABLE_UID = ";
+ query += uidStr;
+ query += " AND REASON = 'M'";
+
+ retcode = findSampleTableCursor1.prepareQuery(query, 0, 4); // no input parms, 1 output col
+ HSLogError(retcode);
+
+ retcode = findSampleTableCursor1.open();
+ HSLogError(retcode);
+
+ // expect at most one row will match the condition because of one
+ // IUS sample table only assumption.
+
+ // sel_stmt.fetch() will set 'tempTabName' to a persistent table name,
+ // if one is found.
+
+ retcode = findSampleTableCursor1.fetch (1, (void *)&tempTabNameUCS2[0]);
+
+ // Handle errors from fetch. If retcode is 100 (meaning no more data),
+ // then set retcode = 0.
+ if (retcode)
{
if (retcode != HS_EOF)
- {
- HSHandleError(retcode);
- }
+ {
+ HSHandleError(retcode);
+ }
else
- {
- retcode = 0;
- }
- break;
+ {
+ retcode = 0;
+ }
}
- else { // fetch() successful.
+ else
+ { // fetch() successful.
char tempTabNameUTF8[2*sizeof(tempTabNameUCS2)];
char *dummyFirstUntranslatedChar;
unsigned int outputDataLen;
@@ -1408,64 +1428,25 @@ Lng32 HSPersSamples::find(HSTableDef *objDef, Int64 &actualRows, NABoolean isEst
HS_ASSERT(rc == 0 && outputDataLen >=0 && outputDataLen < sizeof(tempTabNameUTF8));
tempTabNameUTF8[outputDataLen] = 0;
table = tempTabNameUTF8;
-
- //if (/*!forIUS &&*/ table != "" && actualRows != -1 && isEstimate == FALSE &&
- if ( table != "" && actualRows != -1 && isEstimate == FALSE &&
- ((fabs((sampleRows*100)/percent - actualRows)/actualRows) >
- (CmpCommon::getDefaultLong(USTAT_OBSOLETE_PERCENT_ROWCOUNT) / 100.0f)))
- {
- // The peristent sample table is 'obsolete' - the base table has changed
- // by more than USTAT_OBSOLETE_PERCENT_ROWCOUNT % rows since sample created.
- HSTranMan *TM = HSTranMan::Instance();
- TM->Begin("DROP PERSISTENT SAMPLE TABLE AND REMOVE FROM LIST VIA FIND.");
- // Drop persistent sample table and remove from list.
- NAString ddl = "DROP TABLE " + table;
-
- if (!objDef->isVolatile())
- {
- // alter the table to droppable before the drop
- NAString alterdroppableStmt;
- alterdroppableStmt = "ALTER TABLE ";
- alterdroppableStmt += table ;
- alterdroppableStmt += " DROPPABLE ";
-
- HSFuncExecQuery(alterdroppableStmt, - UERR_INTERNAL_ERROR, NULL,
- HS_QUERY_ERROR, NULL, NULL, TRUE/*doRetry*/ );
- }
-
- HSFuncExecQuery(ddl, - UERR_INTERNAL_ERROR, NULL,
- HS_QUERY_ERROR, NULL, NULL, TRUE/*doRetry*/ );
-
- // If cannot drop table, attempt to remove from list anyway.
- HSCliStatement deletePST(del_stmt,
- (char *)fromTable.data(),
- (char *)table.data());
- retcode = deletePST.execFetch("DELETE " + fromTable );
- if (retcode) TM->Rollback();
- else retcode = TM->Commit();
- if (!retcode) removedObsolete=TRUE;
- }
- else removedObsolete=FALSE;
- }
- } while (!retcode && removedObsolete);
+ }
// Don't overwrite the return code if an error has occurred, but attempt to
// close the cursor anyway (in case it was successfully opened).
if (retcode < 0)
- cursorPST.close();
+ findSampleTableCursor1.close();
else
- retcode = cursorPST.close();
- return retcode;
+ retcode = findSampleTableCursor1.close();
+ return retcode;
}
/***********************************************/
/* METHOD: find() */
/* PURPOSE: finds a persistent sample table for*/
-/* specified reason code.If foundd, */
+/* specified reason code.If found, */
/* the name will be returned in */
/* 'table', or blank if not. */
/* INPUT: uid - source table UID. */
-/* reason - the reason code */
+/* reason - I incremental, M manual */
/* OUTPUT: table - the name of a sample table */
/* found or "" if none found. */
/* requestedRows: requested rows */
@@ -1474,9 +1455,9 @@ Lng32 HSPersSamples::find(HSTableDef *objDef, Int64 &actualRows, NABoolean isEst
/* RETCODE: 0 if no error during search. */
/* non-zero if error */
/***********************************************/
-Lng32 HSPersSamples::find(HSTableDef *objDef, char reason, NAString &table,
- Int64 &requestedRows, Int64 &sampleRows,
- double &sampleRate)
+Lng32 HSPersSamples::find(HSTableDef *objDef, char reason,
+ NAString &table, Int64 &requestedRows,
+ Int64 &sampleRows, double &sampleRate)
{
Lng32 retcode = 0;
NABoolean removedObsolete=FALSE;
@@ -1484,33 +1465,33 @@ Lng32 HSPersSamples::find(HSTableDef *objDef, char reason, NAString &table,
NAWchar tempTabNameUCS2[408]; // first double byte is varchar length + a few extras NAWchars
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
NAString fromTable = persSampTblObjName.getExternalName();
- //double percent; //@ZXunref
- //HSCliStatement::statementIndex del_stmt; //@ZXunref
- HSCliStatement::statementIndex sel_stmt;
+ Int64 uid = objDef->getObjectUID();
+ char uidStr[30];
+ convertInt64ToAscii(uid,uidStr);
// Create query to search for a matching persistent sample in
- // PERSISTENT_SAMPLES table. Persistent samples are matched
- // by the requested sample rowcount, not the actual that was
- // created, since the actual can vary due to VARCHARS, ...
- // A persistent sample is matching if for the same source table
- // and T.REASON = reason_code
- sel_stmt = HSCliStatement::CURSOR_PST_REASON_CODE;
- Int64 uid = objDef->getObjectUID();
- char reason_code = reason;
+ // SB_PERSISTENT_SAMPLES table.
- // sampleRows and allowedDiff are not used
- HSCliStatement cursorPST_reason_code(sel_stmt,
- (char *)fromTable.data(),
- (char *)&uid,
- (char *)&reason_code
- );
- retcode = cursorPST_reason_code.open();
- HSHandleError(retcode);
+ HSCursor findSampleTableCursor(STMTHEAP,"findSampleTable");
+
+ NAString query = "SELECT SAMPLE_NAME, REQUESTED_SAMPLE_ROWS, ACTUAL_SAMPLE_ROWS, SAMPLING_RATIO FROM ";
+ query += fromTable;
+ query += " WHERE TABLE_UID = ";
+ query += uidStr;
+ query += " AND REASON = '";
+ query += reason;
+ query += "'";
+
+ retcode = findSampleTableCursor.prepareQuery(query, 0, 4); // no input parms, 2 output cols
+ HSLogError(retcode);
+
+ retcode = findSampleTableCursor.open();
+ HSLogError(retcode);
// expect at most one row will match the condition because of one
// IUS sample table only assumption.
@@ -1521,7 +1502,7 @@ Lng32 HSPersSamples::find(HSTableDef *objDef, char reason, NAString &table,
// is set to "" and requestdRows, sampleRows and sampleRate
// are not set.
- retcode = cursorPST_reason_code.fetch (4, (void *)&tempTabNameUCS2[0],
+ retcode = findSampleTableCursor.fetch (4, (void *)&tempTabNameUCS2[0],
(void *)&requestedRows,
(void *)&sampleRows,
(void *)&sampleRate
@@ -1563,48 +1544,53 @@ Lng32 HSPersSamples::find(HSTableDef *objDef, char reason, NAString &table,
// Don't overwrite the return code if an error has occurred, but attempt to
// close the cursor anyway (in case it was successfully opened).
if (retcode < 0)
- cursorPST_reason_code.close();
+ findSampleTableCursor.close();
else
- retcode = cursorPST_reason_code.close();
+ retcode = findSampleTableCursor.close();
return retcode;
}
Lng32 HSPersSamples::removeSample(HSTableDef* tabDef, NAString& sampTblName,
- NABoolean useTransaction, const char* txnLabel)
+ char reason, const char* txnLabel)
{
Lng32 retcode = 0;
HSTranMan *TM = HSTranMan::Instance();
if (sampTblName.length() > 0)
{
- if (useTransaction)
- {
- TM->Begin(txnLabel);
- }
-
- HSSample::dropSample(sampTblName, tabDef);
+ TM->Begin(txnLabel);
// Delete row in persistent samples table regardless of whether sample
// could be dropped.
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
NAString fromTable = persSampTblObjName.getExternalName();
- HSCliStatement::statementIndex del_stmt = HSCliStatement::DELETE_PST;
- HSCliStatement deletePST(del_stmt,
- (char*)fromTable.data(),
- (char*)sampTblName.data());
- retcode = deletePST.execFetch("DELETE " + fromTable );
- if (useTransaction)
- {
- if (retcode)
- TM->Rollback();
- else
- TM->Commit();
- }
+ Int64 objUID = tabDef->getObjectUID();
+
+ NAString dml((size_t)500/*allocate 500 bytes*/, STMTHEAP);
+ dml.append("DELETE FROM ");
+ dml += fromTable; // in UTF8
+ dml += " WHERE TABLE_UID = ";
+ dml += Int64ToNAString(objUID);
+ dml += " AND REASON = '";
+ dml += reason;
+ dml += "'";
+
+ retcode = HSFuncExecQuery(dml, - UERR_INTERNAL_ERROR, NULL,
+ HS_QUERY_ERROR, NULL, NULL, TRUE/*doRetry*/ );
+ HSHandleError(retcode);
+
+ HSSample::dropSample(sampTblName, tabDef);
+
+ if (retcode)
+ TM->Rollback();
+ else
+ TM->Commit();
+
}
return retcode;
@@ -1618,7 +1604,7 @@ Lng32 HSPersSamples::removeSample(HSTableDef* tabDef, NAString& sampTblName,
/* can be determined from statistics. */
/* INPUTS: tabDef - a pointer to table struct. */
/* isEstimate - TRUE if 'actualRows' is est. on input. */
-/* isManual - sets REASON in PERSISTENT_SAMPLES. */
+/* isManual - sets REASON in SB_PERSISTENT_SAMPLES. */
/* IN/OUTS: sampleRows - the desired size of sample on input, */
/* actual sample size on output. */
/* actualRows - table size (possibly an est.) on input */
@@ -1644,20 +1630,19 @@ Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
if (!tabDef || tabDef->setHasSyskeyFlag() != 0) return -1;
// Sets flag in tabDef indicating primary key is SYSKEY.
- // Drop the prior IUS persistent sample table if it exists, and remove
- // the corresponding row from the PERSISTENT_SAMPLES table.
- if (reason == 'I')
- {
- Int64 dummy1, dummy2;
- double dummy3;
- NAString oldSampTblName;
- retcode = find(tabDef, char('I'), oldSampTblName,
- dummy1, dummy2, dummy3);
- removeSample(tabDef, oldSampTblName, TRUE,
- "DROP OLD IUS PERSISTENT SAMPLE TABLE AND REMOVE FROM LIST");
- }
+ // Drop the prior persistent sample table if it exists, and remove
+ // the corresponding row from the SB_PERSISTENT_SAMPLES table.
+
+ Int64 dummy1, dummy2;
+ double dummy3;
+ NAString oldSampTblName;
+ retcode = find(tabDef, reason, oldSampTblName,
+ dummy1, dummy2, dummy3);
+ if (retcode == 0)
+ removeSample(tabDef, oldSampTblName, reason,
+ "DROP OLD PERSISTENT SAMPLE TABLE AND REMOVE FROM LIST");
- // Save original requested number of sample rows for inserting to PERSISTENT_SAMPLES.
+ // Save original requested number of sample rows for inserting to SB_PERSISTENT_SAMPLES.
reqSampleRows = sampleRows;
Lng32 st = (CmpCommon::getDefault(USTAT_IUS_USE_PERIODIC_SAMPLING) == DF_ON) ?
@@ -1673,7 +1658,6 @@ Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
retcode = sample.make(isEstimate, sampleName, actualRows, sampleRows, TRUE,
reason != 'I', /* partitioned for IUS */
- createDandI,
minRowCtPerPartition
);
// sampleName output & actualRows will get modified if necessary
@@ -1682,72 +1666,22 @@ Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
{
// Sample table successfully created, insert entry into
- // PERSISTENT_SAMPLES table. If unable to insert entry,
+ // SB_PERSISTENT_SAMPLES table. If unable to insert entry,
// drop sample table.
stmt = HSCliStatement::INSERT_PST;
percent = ((float)sampleRows/(float)actualRows)*100;
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
into_table = persSampTblObjName.getExternalName();
objUID = tabDef->getObjectUID();
- TM->Begin("INSERT INTO PERSISTENT_SAMPLES TABLE.");
+ TM->Begin("INSERT INTO SB_PERSISTENT_SAMPLES TABLE.");
char timeStr[HS_TIMESTAMP_SIZE];
hs_formatTimestamp(timeStr);
-#if 1 // Use the static INSERT_PST query defined in w:/ustat/sqlhist.mdf
- NAWString *pSampleNameInUTF16 =
- charToUnicode ( (Lng32) CharInfo::UTF8 // char set of src str
- , sampleName/*in_utf8*/.data() // src str
- , (Int32)sampleName/*in_utf8*/.length() // src str len in bytes
- , STMTHEAP // heap for allocated target str
- );
- NAWString sampleNameInUTF16(STMTHEAP);
- if (pSampleNameInUTF16 != NULL && pSampleNameInUTF16->length() > 0)
- sampleNameInUTF16.append(*pSampleNameInUTF16);
- delete pSampleNameInUTF16; pSampleNameInUTF16 = NULL;
- if (!sampleName.isNull() && sampleNameInUTF16.length() <= 0)
- {
- NAString str0((size_t)600/*allocate 600 bytes*/, STMTHEAP);
- str0.append("INSERT ");
- str0.append(sampleName);
- *CmpCommon::diags() << DgSqlCode(-UERR_GENERIC_ERROR)
- << DgString0(str0.data())
- << DgString1("-2109")
- << DgString2("Unable to translate sample name from UTF8 to UCS2");
- // HSHandleError(-UERR_GENERIC_ERROR);
- return (-UERR_GENERIC_ERROR);
- }
-
- HSGlobalsClass *hs_globals = GetHSContext();
-
- NAWString V1(L"Dummy value for V1 column", STMTHEAP);
- NAWString V2(L"Dummy value for V2 column", STMTHEAP);
- // Set the UPDATE_DATE to the epoch time '1970-01-01 00:00:00'.
- // See its usage in HSGlobalsClass::begin_IUS_work() and ::end_IUS_work()
- HSCliStatement insertPST(stmt,
- (char *)into_table.data(),
- (char *)&objUID,
- (char *)&reqSampleRows,
- (char *)&sampleRows,
- (char *)&percent,
- (char *)&timeStr[0],
- (char *)&reason,
- (char *)sampleNameInUTF16.data(),
- (char *)L"", // IUS_SEARCH_CONDITION (added after IUS stmt)
- (char *)"1970-01-01 00:00:00", // epoch UPDATE_DATE
- (char *)"", // dummy IUS_UPDATE_HISTORY
- (char *)V1.data(), // V1
- (char *)V2.data()); // V2
-
- retcode = insertPST.execFetch("INSERT " + sampleName );
- HSHandleError(retcode);
-
-
-#else // Use dynamic query as a workaround if having problem with the static query
NAString dml((size_t)500/*allocate 500 bytes*/, STMTHEAP);
dml.append("INSERT INTO ");
dml += into_table; // in UTF8
@@ -1770,31 +1704,21 @@ Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
, TRUE // in - NABoolean encloseInQuotes = TRUE
);
dml += quotedSampleName; // in UTF8
+ HSGlobalsClass *hs_globals = GetHSContext();
ToQuotedString ( hs_globals->getWherePredicateForIUS() // out - NAString "edStr
, "" // in - const NAString &internalStr
, TRUE // in - NABoolean encloseInQuotes = TRUE
);
- dml += "',_UCS2''"; // IUS_SEARCH_CONDITION (added after IUS stmt)
- dml += ",TIMESTAMP'";
- dml += timeStr; // UPDATE_DATE TIMESTAMP
- dml += ",";
- dml += ""; // IUS_UPDATE_HISTORY
- NAWString quotedEmptyWStr(STMTHEAP);
- ToQuotedString ( quotedEmptyWStr // out - NAString "edStr
- , "" // in - const NAString &internalStr
- , TRUE // in - NABoolean encloseInQuotes = TRUE
- );
- dml += "',_UCS2";
- dml += quotedEmptyWStr; // V1
- dml += "',_UCS2";
- dml += quotedEmptyWStr; // V2
+ dml += ",_UCS2''"; // IUS_SEARCH_CONDITION (added after IUS stmt)
+ dml += ",TIMESTAMP'1970-01-01 00:00:00'"; // UPDATE_START_TIME TIMESTAMP
+ dml += ",''"; // UPDATER_INFO
+ dml += ",_UCS2''"; // V1
+ dml += ",_UCS2''"; // V2
dml += ");";
retcode = HSFuncExecQuery(dml, - UERR_INTERNAL_ERROR, NULL,
HS_QUERY_ERROR, NULL, NULL, TRUE/*doRetry*/ );
HSHandleError(retcode);
-#endif
-
if (!retcode) TM->Commit();
else {
@@ -1818,22 +1742,11 @@ Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
return retcode;
}
-Lng32 HSPersSamples::createAndInsert(HSTableDef *tabDef, NAString &sampleName,
- Int64 &sampleRows, Int64 &actualRows,
- NABoolean isEstimate, NABoolean isManual,
- Int64 minRowCtPerPartition)
- {
- char reason = (isManual == TRUE) ? 'M' : 'A';
- return createAndInsert(tabDef, sampleName, sampleRows, actualRows, isEstimate, reason,
- FALSE, // do not create DandI
- minRowCtPerPartition
- );
- }
-
/***********************************************/
/* METHOD: remove() */
-/* PURPOSE: Remove all persistent sample tables*/
-/* for the table with object uid 'uid'*/
+/* PURPOSE: Remove all manual (i.e., not IUS) */
+/* persistent sample tables for */
+/* the table with object uid 'uid' */
/* and within 'allowedDiff' fraction */
/* of 'sampleRows' in size. */
/* INPUT: uid - source table UID. */
@@ -1860,17 +1773,16 @@ Lng32 HSPersSamples::removeMatchingSamples(HSTableDef *tabDef,
HSTranMan *TM = HSTranMan::Instance();
Lng32 retcode = 0;
NAString ddl, table, fromTable;
- HSCliStatement::statementIndex stmt;
+ NABoolean nothingToDrop = TRUE;
TM->Begin("DROP PERSISTENT SAMPLE TABLE AND REMOVE FROM LIST.");
// Loop until all persistent samples matching criteria have been removed.
Int64 actualRows = -1; // Obsolete samples will not be removed by find().
NABoolean isEstimate = TRUE;
retcode = find(tabDef, actualRows, isEstimate, sampleRows, allowedDiff, table);
- stmt = HSCliStatement::DELETE_PST;
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
@@ -1878,19 +1790,27 @@ Lng32 HSPersSamples::removeMatchingSamples(HSTableDef *tabDef,
while (retcode == 0 && table != "")
{
// Drop persistent sample table and remove from list.
- retcode = removeSample(tabDef, table, FALSE, "");
+ nothingToDrop = FALSE;
+ retcode = removeSample(tabDef, table, 'M', "");
if (!retcode)
retcode = find(tabDef, actualRows, isEstimate, sampleRows, allowedDiff, table);
}
if (retcode) TM->Rollback();
else retcode = TM->Commit();
+
+ if (nothingToDrop)
+ {
+ // we didn't drop anything; warn the user
+ *CmpCommon::diags() << DgSqlCode(UERR_WARNING_NO_SAMPLE_TABLE);
+ }
+
return retcode;
}
/*********************************************************************/
/* METHOD: readIUSUpdateInfo() */
-/* PURPOSE: Retrieve the UPDATE_DATE and IUS_UPDATE_HISTORY columns */
+/* PURPOSE: Retrieve the UPDATE_START_TIME and UPDATER_INFO columns */
/* of the persistent samples table for the row identified */
/* by the UID of the passed HSTableDef. */
/* INPUT: tblDef - Ptr to HSTableDef for table the sample is on. */
@@ -1907,47 +1827,104 @@ Lng32 HSPersSamples::readIUSUpdateInfo(HSTableDef* tblDef,
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
NAString fromTable = persSampTblObjName.getExternalName();
- HSCliStatement::statementIndex sel_stmt;
+ Int64 uid = tblDef->getObjectUID();
+ char uidStr[30];
+ convertInt64ToAscii(uid,uidStr);
+ HSCursor readIUSInfoCursor(STMTHEAP,"readIUSUpdateInfo");
- sel_stmt = HSCliStatement::CURSOR_PST_UPDATE_INFO;
+ NAString query = "SELECT UPDATER_INFO, "
+ "CAST(UPDATE_START_TIME - TIMESTAMP '1970-01-01 00:00:00' AS LARGEINT) FROM ";
+ query += fromTable;
+ query += " WHERE TABLE_UID = ";
+ query += uidStr;
+ query += " AND REASON = 'I'";
- Int64 uid = tblDef->getObjectUID();
- HSCliStatement cursorPSTUpdInfo(sel_stmt,
- (char*)fromTable.data(),
- (char*)&uid);
- retcode = cursorPSTUpdInfo.open();
- HSHandleError(retcode);
+ retcode = readIUSInfoCursor.prepareQuery(query, 0, 2); // no input parms, 2 output cols
+ HSLogError(retcode);
+
+ retcode = readIUSInfoCursor.open();
+ HSLogError(retcode);
- retcode = cursorPSTUpdInfo.fetch(2, (void*)updTimestamp, (void*)updHistory);
+ if (retcode == 0)
+ {
+ struct
+ {
+ short len;
+ char data[129]; // to fetch a varchar; UPDATER_INFO is VARCHAR(128)
+ } buffer;
+
+ retcode = readIUSInfoCursor.fetch(2, (void *)&buffer, (void *)updTimestamp);
+ HSLogError(retcode);
+ if (retcode == 0)
+ {
+ memmove(updHistory,buffer.data,buffer.len);
+ updHistory[buffer.len] = '\0'; // to insure a null terminator
+ }
+ }
// Don't overwrite a nonzero return code (error, warning, or HS_EOF) from
// the fetch, but attempt to close the cursor anyway.
if (retcode != 0)
- cursorPSTUpdInfo.close();
+ readIUSInfoCursor.close();
else
- retcode = cursorPSTUpdInfo.close();
+ retcode = readIUSInfoCursor.close();
return retcode;
}
+
+// helper function that doubles any single quotes in text, so that
+// we can write SQL text into a SQL table
+void doubleUpSingleQuotes(const char *text, NAString & result)
+ {
+ const char * next = text;
+ char buf[2];
+ char doubledSingleQuote[3] = "''";
+
+ buf[1] = '\0';
+
+ while (*next)
+ {
+ if (*next == '\'')
+ result += doubledSingleQuote; // double up any single quote
+ else
+ {
+ // NAString += with a char doesn't work; have to use a char[]
+ buf[0] = *next;
+ result += buf;
+ }
+ next++;
+ }
+ }
+
+
/*********************************************************************/
/* METHOD: updIUSUpdateInfo() */
-/* PURPOSE: Update the UPDATE_DATE and IUS_UPDATE_HISTORY columns of */
+/* PURPOSE: Update the UPDATE_START_TIME and UPDATER_INFO columns of */
/* the persistent samples table for the row identified by */
/* the UID of the passed HSTableDef. */
/* INPUT: tblDef - Ptr to HSTableDef for table the sample is on. */
-/* updHistory - Buffer containing update history value. */
+/* updHistory - Buffer containing updater info. */
/* updTimestamp - Char representation of update timestamp. */
+/* updWhereCondition - if not null, the where predicate of */
+/* the last completed IUS operation. (If null, we don't */
+/* update this column.) */
+/* requestedSampleRows - if non-null, points to expected # */
+/* of rows in sample table (based on sampling rate). */
+/* actualSampleRows - if non-null, actual # of rows. */
/* RETCODE: Status code from the update operation. */
/*********************************************************************/
Lng32 HSPersSamples::updIUSUpdateInfo(HSTableDef* tblDef,
- char* updHistory,
- char* updTimestampStr)
+ const char* updHistory,
+ const char* updTimestampStr,
+ const char* updWhereCondition,
+ const Int64* requestedSampleRows,
+ const Int64* actualSampleRows)
{
Lng32 retcode = 0;
HSErrorCatcher errorCatcher(retcode, - UERR_INTERNAL_ERROR, "updIUSUpdateInfo", TRUE);
@@ -1961,19 +1938,87 @@ Lng32 HSPersSamples::updIUSUpdateInfo(HSTableDef* tblDef,
// table for the passed table UID.
ComObjectName persSampTblObjName(*catalog_,
*schema_,
- "PERSISTENT_SAMPLES",
+ "SB_PERSISTENT_SAMPLES",
COM_TABLE_NAME,
ComAnsiNamePart::INTERNAL_FORMAT,
STMTHEAP);
NAString updTable = persSampTblObjName.getExternalName();
- HSCliStatement::statementIndex updStmt = HSCliStatement::UPDATE_PST_UPDATE_INFO;
- HSCliStatement updPST(updStmt,
- (char*)updTable.data(),
- updTimestampStr,
- updHistory,
- (char*)&tblUID);
- retcode = updPST.execFetch("UPDATE " + updTable );
+ Int64 uid = tblDef->getObjectUID();
+ char buf[30];
+
+ HSCursor writeIUSInfoCursor(STMTHEAP,"writeIUSUpdateInfo");
+
+ // The caller should have checked to see that the row existed first,
+ // but even so it is possible (though unlikely) that someone could
+ // have deleted the row in the meantime.
+
+ NAString query = "UPDATE ";
+ query += updTable;
+ query += " SET UPDATE_START_TIME = TIMESTAMP '";
+ query += updTimestampStr;
+ query += "', UPDATER_INFO = '";
+ query += updHistory;
+ query += "'";
+ if (updWhereCondition)
+ {
+ NAString doubledUp;
+ doubleUpSingleQuotes(updWhereCondition,doubledUp /* out*/);
+ query += ", LAST_WHERE_PREDICATE = ";
+ if (strlen(updWhereCondition) > 250)
+ {
+ // let SQL deal with character truncation issues
+ query += "SUBSTRING('";
+ query += doubledUp;
+ query += "' FOR 250)";
+ }
+ else
+ {
+ query += "'";
+ query += doubledUp;
+ query += "'";
+ }
+ }
+ if (requestedSampleRows)
+ {
+ convertInt64ToAscii(*requestedSampleRows, buf);
+ query += ", REQUESTED_SAMPLE_ROWS = ";
+ query += buf;
+ }
+ if (actualSampleRows)
+ {
+ convertInt64ToAscii(*actualSampleRows, buf);
+ query += ", ACTUAL_SAMPLE_ROWS = ";
+ query += buf;
+ }
+ query += " WHERE TABLE_UID = ";
+ convertInt64ToAscii(uid,buf);
+ query += buf;
+ query += " AND REASON = 'I'";
+
+ retcode = writeIUSInfoCursor.prepareQuery(query, 0, 0);
+ HSLogError(retcode);
+ retcode = writeIUSInfoCursor.open();
+ HSLogError(retcode);
+
+ if (retcode == 0)
+ {
+ retcode = writeIUSInfoCursor.fetch(0,0);
+ if (retcode == 100) // a successful UPDATE returns 100 on fetch
+ retcode = 0;
+ HSLogError(retcode);
+ }
+
+ // Don't overwrite a nonzero return code (error, warning) from
+ // the fetch, but attempt to close the cursor anyway.
+ if (retcode != 0)
+ writeIUSInfoCursor.close();
+ else
+ {
+ retcode = writeIUSInfoCursor.close();
+ if (retcode == 100) // a successful UPDATE returns 100 on close too
+ retcode = 0;
+ }
if (retcode)
TM->Rollback();
@@ -2493,7 +2538,8 @@ HSCursor::HSCursor(NAHeap *heap, const char* stmtName)
colDesc_(NULL), numEntries_(0),
ptrNAType_(NULL), dataBuf_(NULL), outputDataLen_(0),
group_(NULL), boundaryRowSet_(NULL), rowset_fields_(NULL),
- closeStmtNeeded_(FALSE), retcode_(0), heap_(heap)
+ closeStmtNeeded_(FALSE), retcode_(0), heap_(heap),
+ lastFetchReturned100_(FALSE)
{
static SQLMODULE_ID module;
static NABoolean moduleSet = FALSE;
@@ -5817,6 +5863,45 @@ Lng32 printPlan(SQLSTMT_ID *stmt)
#endif // NA_USTAT_USE_STATIC not defined
+// Use a query on the Explain virtual table to see if the plan for stmt uses
+// MDAM, and include this information in the ulog. This function is called
+// only if logging is turned on.
+Lng32 checkMdam(SQLSTMT_ID *stmt)
+{
+ Lng32 retcode = 0;
+ HSLogMan *LM = HSLogMan::Instance();
+
+ NAString stmtStr = "select cast(count(*) as int) from table(explain(null,'";
+ stmtStr.append((char*)stmt->identifier)
+ .append("')) where description like '%mdam%'");
+
+ HSCursor cursor(STMTHEAP, "HS_CLI_CHECK_MDAM");
+ retcode = cursor.prepareQuery(stmtStr.data(), 0, 1);
+ HSHandleError(retcode);
+ SQLSTMT_ID* stmtId = cursor.getStmt();
+ SQLDESC_ID* outputDesc = cursor.getOutDesc();
+ retcode = cursor.open();
+ HSHandleError(retcode);
+
+ Int32 rowCount = 0;
+ retcode = SQL_EXEC_Fetch(stmtId, outputDesc, 1, &rowCount, NULL);
+ if (retcode == 0)
+ {
+ if (rowCount > 0)
+ LM->Log("MDAM is used for this query.");
+ else
+ LM->Log("MDAM is NOT used for this query.");
+ }
+ else if (retcode == 100)
+ LM->Log("MDAM check query returned no rows.");
+ else
+ {
+ LM->Log("MDAM check query failed.");
+ HSLogError(retcode);
+ }
+
+ return retcode;
+}
/***********************************************/
/* METHOD: getRowCountFromStats(Int64* ) */
@@ -6127,13 +6212,14 @@ Lng32 HSCursor::prepareQuery(const char *cliStr, Lng32 numParams, Lng32 numResul
}
//@ZXnew
-Lng32 HSCursor::open(Lng32 numParams)
+Lng32 HSCursor::open(Lng32 numParams, void * in01)
{
Lng32 retcode = SQL_EXEC_ClearDiagnostics(stmt_);
HSHandleError(retcode);
- retcode = SQL_EXEC_Exec(stmt_, inputDesc_, numParams);
+ retcode = SQL_EXEC_Exec(stmt_, inputDesc_, numParams, in01, NULL);
HSHandleError(retcode);
closeStmtNeeded_ = TRUE;
+ lastFetchReturned100_ = FALSE;
return retcode;
}
@@ -6149,7 +6235,7 @@ Lng32 HSCursor::fetch(Lng32 numResults,
void * out22, void * out23, void * out24,
void * out25)
{
- HS_ASSERT(numResults > 0 && out01 != NULL);
+ HS_ASSERT((numResults == 0) || (numResults > 0 && out01 != NULL));
Lng32 retcode = SQL_EXEC_Fetch(stmt_, outputDesc_, numResults,
out01, NULL, out02, NULL, out03, NULL,
out04, NULL, out05, NULL, out06, NULL,
@@ -6160,6 +6246,29 @@ Lng32 HSCursor::fetch(Lng32 numResults,
out19, NULL, out20, NULL, out21, NULL,
out22, NULL, out23, NULL, out24, NULL,
out25, NULL);
+ lastFetchReturned100_ = (retcode == 100);
+ return retcode;
+}
+
+Lng32 HSCursor::close()
+{
+ // This is done by the dtor ordinarily. However, if a cursor needs to
+ // be reused (e.g., different input parameters), we need to be able
+ // to close without destroying the object.
+
+ Lng32 retcode = 0;
+ if (closeStmtNeeded_)
+ {
+ if (lastFetchReturned100_)
+ {
+ // if the last fetch simply ran to the end of the cursor, reset
+ // the diags area so we don't return the 100 again on the close
+ SQL_EXEC_ClearDiagnostics(stmt_);
+ }
+ retcode = SQL_EXEC_CloseStmt(stmt_);
+ closeStmtNeeded_ = FALSE; // always set to FALSE so we don't try again
+ }
+ lastFetchReturned100_ = FALSE;
return retcode;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_cli.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_cli.h b/core/sql/ustat/hs_cli.h
index 3c8bb01..3d91ac4 100644
--- a/core/sql/ustat/hs_cli.h
+++ b/core/sql/ustat/hs_cli.h
@@ -75,6 +75,7 @@ Lng32 HSFuncExecQuery( const char *dml
, const HSTableDef *tabDef = NULL
, NABoolean doRetry = FALSE
, short errorToIgnore = 0
+ , NABoolean checkMdam = FALSE
);
Lng32 HSFuncExecDDL( const char *dml
@@ -92,6 +93,7 @@ Lng32 CreateHistView (const HSGlobalsClass* hsGlobal);
// Drop the sample table.
Lng32 DropSampleTable();
+Lng32 checkMdam(SQLSTMT_ID *stmt);
Lng32 printPlan(SQLSTMT_ID *stmt);
void getRowCountFromStats(Int64 * rowsAffected , const HSTableDef *tabDef = NULL);
@@ -175,14 +177,11 @@ class HSSample
NABoolean unpartitioned=FALSE, //input - used to specify if the
// method is being called to create
// unpartitioned persistent sample
- NABoolean createDandI=FALSE, //input - if table D and I
- // should be created
Int64 minRowCtPerPartition = -1 // minimal row RC per partition
);
// Create sample table (called by make).
Lng32 create(NABoolean unpartitioned = FALSE,
- NABoolean isPersSample = FALSE,
- NABoolean createDandI = FALSE
+ NABoolean isPersSample = FALSE
);
Lng32 create(NAString& tblName,
NABoolean unpartitioned = FALSE,
@@ -414,18 +413,22 @@ class HSPersSamples
public:
// Creates or returns instance.
static HSPersSamples* Instance(const NAString &catalog,
- NABoolean createTable = TRUE);
-
- static Lng32 updIUSUpdateInfo(HSTableDef* tblDef,
- char* updHistory,
- char* updTimestampStr);
-
- static Lng32 readIUSUpdateInfo(HSTableDef* tblDef,
- char* updHistory,
- Int64* updTimestamp);
-
- // finds a persistent sample table for UID and reason code and returns in 'table'.
- Lng32 find(HSTableDef *tabDef, char reason_code, NAString &table,
+ const NAString &schema);
+
+ Lng32 updIUSUpdateInfo(HSTableDef* tblDef,
+ const char* updHistory,
+ const char* updTimestampStr,
+ const char* updWhereCondition,
+ const Int64* requestedSampleRows = NULL,
+ const Int64* actualSampleRows = NULL);
+
+ Lng32 readIUSUpdateInfo(HSTableDef* tblDef,
+ char* updHistory,
+ Int64* updTimestamp);
+
+ // finds a persistent sample table for UID and reason code and returns it in 'table'.
+ // (returns ' ' in table if none is found).
+ Lng32 find(HSTableDef *tabDef, char reason, NAString &table,
Int64 &requestedRows, Int64 &sampleRows, double &sampleRate);
// finds a persistent sample table for UID and sample size and returns in 'table'.
@@ -439,10 +442,6 @@ class HSPersSamples
NABoolean isEstimate, char reason,
NABoolean createDandI=FALSE,
Int64 minRowsCtPerPartition = -1);
- Lng32 createAndInsert(HSTableDef *tabDef, NAString &sampleName,
- Int64 &sampleRows, Int64 &actualRows,
- NABoolean isEstimate, NABoolean isManual,
- Int64 minRowsCtPerPartition = -1);
// remove persistent sample table(s) based on uid, sampleRows, and the
// allowed difference between the number of rows and sampleRows.
@@ -451,15 +450,24 @@ class HSPersSamples
// drop the named sample table and remove its entry from the
// PERSISTENT_SAMPLES table.
Lng32 removeSample(HSTableDef* tabDef, NAString& sampTblName,
- NABoolean useTransaction, const char* txnLabel);
+ char reason, const char* txnLabel);
- protected:
- HSPersSamples(); /* ensure only 1 instance of class */
+ ~HSPersSamples();
+
+ protected: /* ensure only 1 instance of class */
+
+ HSPersSamples(const NAString &catalog,
+ const NAString &schema);
+
+ void setCatalogSchema(const NAString &catalog,
+ const NAString &schema);
+
private:
+
static THREAD_P HSPersSamples* instance_; /* 1 and only 1 instance */
- static THREAD_P NAList<NAString>* persSampleTablesList_;
- static THREAD_P NAString* catalog_;
- static THREAD_P NAString* schema_;
+ NAString* catalog_;
+ NAString* schema_;
+ NABoolean triedCreatingSBPersistentSamples_;
};
/*****************************************************************************/
@@ -492,6 +500,7 @@ class HSPersData
#define HS_MODULE "HP_SYSTEM_CATALOG.SYSTEM_SCHEMA.SQLHIST_N29_000"
#define HS_MODULE_LENGTH 50 // more than needed
#define HS_STMTID "HS_CLI_DYNSTMT"
+#define HS_INTERVAL_STMT_ID "HS_INTERVAL_STMT_ID"
#define HS_STMTID_LENGTH 50
#define HS_FUNC_EXEC_QUERY_STMTID "HS_FUNC_EXEC_QUERY_DYNSTMT"
@@ -884,7 +893,7 @@ public:
// Functions below used when the HSCursor is instantiated for a dynamic query.
Lng32 prepareQuery(const char *cliStr, Lng32 numParams, Lng32 numResults);
- Lng32 open(Lng32 numParams = 0);
+ Lng32 open(Lng32 numParams = 0, void *in01 = NULL);
Lng32 fetch(Lng32 numResults,
void* out01, void* out02 = NULL, void* out03 = NULL,
void* out04 = NULL, void* out05 = NULL, void* out06 = NULL,
@@ -895,20 +904,8 @@ public:
void* out19 = NULL, void* out20 = NULL, void* out21 = NULL,
void* out22 = NULL, void* out23 = NULL, void* out24 = NULL,
void* out25 = NULL);
- Lng32 close()
- {
- // This is done by the dtor ordinarily. This function exists just to avoid
- // the need for ifdefs around calls to close() on a variable that is either
- // an HSCliStatement or HSCursor depending on the value of NA_USTAT_USE_STATIC.
- Lng32 retcode = 0;
- if (closeStmtNeeded_)
- {
- retcode = SQL_EXEC_CloseStmt(stmt_);
- if (!retcode)
- closeStmtNeeded_ = FALSE;
- }
- return retcode;
- }
+ Lng32 close();
+
SQLSTMT_ID* getStmt() {return stmt_;}
SQLDESC_ID* getOutDesc() {return outputDesc_;}
SQLDESC_ID* getInDesc() {return inputDesc_;}
@@ -972,6 +969,8 @@ private:
NAHeap *heap_;
+ NABoolean lastFetchReturned100_;
+
// prepare a dynamic sql statement.
Lng32 prepare(const char *cliStr, const Lng32 outDescEntries = 500);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/021ebd6c/core/sql/ustat/hs_const.h
----------------------------------------------------------------------
diff --git a/core/sql/ustat/hs_const.h b/core/sql/ustat/hs_const.h
index 9c0bf7d..5dbbb57 100644
--- a/core/sql/ustat/hs_const.h
+++ b/core/sql/ustat/hs_const.h
@@ -153,7 +153,8 @@ enum USTAT_ERROR_CODES {UERR_SYNTAX_ERROR = 15001,
UERR_WARNING_IUS_TOO_MUCH_UEC_CHANGE_TOTAL = 9225,
UERR_WARNING_IUS_ZERO_UEC_INTERVAL = 9226,
UERR_WARNING_IUS_EMPTY_INTERVAL = 9227,
- UERR_IUS_NO_SUFFICIENT_MEMORY = 9230,
+ UERR_WARNING_NO_SAMPLE_TABLE = 9228,
+ UERR_WARNING_IUS_INSUFFICIENT_MEMORY = 9230,
UERR_IUS_WRONG_RANDOM = 9231,
UERR_IUS_IN_PROGRESS = 9232,
UERR_WARNING_IUS_WHERE_CLAUSE_TOO_LONG = 9233,
@@ -168,10 +169,14 @@ enum USTAT_ERROR_CODES {UERR_SYNTAX_ERROR = 15001,
UERR_CANT_CREATE_HIVE_STATS_SCHEMA = 9242,
UERR_YOU_WILL_LIKELY_BE_SORRY = 9243,
UERR_USER_TRANSACTION = 9244,
+ UERR_UNEXPECTED_BACKWARDS_DATA = 9245,
UERR_LOB_STATS_NOT_SUPPORTED = 9246,
UERR_VOLATILE_TABLES_NOT_SUPPORTED = 9247,
- UERR_NO_ERROR = 9250,
- UERR_LAST_ERROR = 9250
+ UERR_FASTSTATS_MEM_ALLOCATION_ERROR = 9248,
+ UERR_IUS_IS_DISABLED = 9249,
+ UERR_WARNING_IUS_NO_LONGER_ALL_NULL = 9250,
+ UERR_NO_ERROR = 9259,
+ UERR_LAST_ERROR = 9259
};
// reason of a histogram entry. it is a char value.