You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by an...@apache.org on 2016/05/31 22:53:41 UTC
[6/8] incubator-trafodion git commit: hive data modification
detection: commit #3
hive data modification detection: commit #3
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/b1a8f024
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/b1a8f024
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/b1a8f024
Branch: refs/heads/master
Commit: b1a8f024b5a3d4db368b51bbd2b6aac7035b9571
Parents: fa70e68
Author: Anoop Sharma <an...@esgyn.com>
Authored: Mon May 30 13:48:16 2016 +0000
Committer: Anoop Sharma <an...@esgyn.com>
Committed: Mon May 30 13:48:16 2016 +0000
----------------------------------------------------------------------
core/sql/comexe/ComTdbFastTransport.cpp | 3 +-
core/sql/comexe/ComTdbFastTransport.h | 5 +-
core/sql/executor/ExFastTransport.cpp | 87 +++++++++--
core/sql/executor/ExFastTransport.h | 5 +
core/sql/exp/ExpLOBaccess.cpp | 155 +++++++++++++++++---
core/sql/exp/ExpLOBaccess.h | 4 +-
core/sql/generator/GenFastTransport.cpp | 60 +++++---
core/sql/optimizer/HDFSHook.cpp | 3 +-
core/sql/regress/hive/EXPECTED003 | 9 --
core/sql/regress/hive/EXPECTED005 | 8 +-
core/sql/regress/hive/EXPECTED006 | 3 -
core/sql/regress/hive/EXPECTED018 | 208 +++++++++++++++------------
core/sql/regress/hive/TEST003 | 3 -
core/sql/regress/hive/TEST006 | 1 -
core/sql/regress/hive/TEST018 | 4 +-
15 files changed, 386 insertions(+), 172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/comexe/ComTdbFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbFastTransport.cpp b/core/sql/comexe/ComTdbFastTransport.cpp
index d995fdb..f492e8f 100644
--- a/core/sql/comexe/ComTdbFastTransport.cpp
+++ b/core/sql/comexe/ComTdbFastTransport.cpp
@@ -98,7 +98,8 @@ ComTdbFastExtract::ComTdbFastExtract(
hdfsIOBufferSize_(hdfBuffSize),
hdfsReplication_(replication),
ioTimeout_(ioTimeout),
- childDataRowLen_(childDataRowLen)
+ childDataRowLen_(childDataRowLen),
+ modTSforDir_(-1)
{
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/comexe/ComTdbFastTransport.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbFastTransport.h b/core/sql/comexe/ComTdbFastTransport.h
index 1ae7625..98e93eb 100644
--- a/core/sql/comexe/ComTdbFastTransport.h
+++ b/core/sql/comexe/ComTdbFastTransport.h
@@ -401,6 +401,8 @@ public:
return childDataRowLen_;
}
+ void setModTSforDir(Int64 v) { modTSforDir_ = v; }
+ Int64 getModTSforDir() const { return modTSforDir_; }
protected:
NABasicPtr targetName_; // 00 - 07
@@ -427,9 +429,10 @@ protected:
UInt16 ioTimeout_; // 128 - 129
UInt16 filler_; // 130 - 131
UInt32 childDataRowLen_; // 132 - 135
+ Int64 modTSforDir_; // 136 - 143
// Make sure class size is a multiple of 8
- char fillerComTdbFastTransport_[8]; // 136 - 143
+ char fillerComTdbFastTransport_[8]; // 144 - 151
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/executor/ExFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp
index b881b14..45c3959 100644
--- a/core/sql/executor/ExFastTransport.cpp
+++ b/core/sql/executor/ExFastTransport.cpp
@@ -89,6 +89,7 @@ ExFastExtractTcb::ExFastExtractTcb(
, sourceFieldsConvIndex_(NULL)
, currBuffer_(NULL)
, bufferAllocFailuresCount_(0)
+ , modTS_(-1)
{
ex_globals *stmtGlobals = getGlobals();
@@ -482,6 +483,16 @@ Lng32 ExHdfsFastExtractTcb::lobInterfaceCreate()
}
+Lng32 ExHdfsFastExtractTcb::lobInterfaceDataModCheck()
+{
+ return ExpLOBinterfaceDataModCheck(lobGlob_,
+ targetLocation_,
+ hdfsHost_,
+ hdfsPort_,
+ myTdb().getModTSforDir(),
+ 0);
+}
+
Lng32 ExHdfsFastExtractTcb::lobInterfaceClose()
{
@@ -530,13 +541,13 @@ Int32 ExHdfsFastExtractTcb::fixup()
ex_tcb::fixup();
-
if(!myTdb().getSkipWritingToFiles() &&
!myTdb().getBypassLibhdfs())
ExpLOBinterfaceInit
(lobGlob_, getGlobals()->getDefaultHeap(),TRUE);
+ modTS_ = myTdb().getModTSforDir();
return 0;
}
@@ -681,9 +692,62 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
{
case EXTRACT_NOT_STARTED:
{
+ pstate.step_= EXTRACT_CHECK_MOD_TS;
+ }
+ break;
+
+ case EXTRACT_CHECK_MOD_TS:
+ {
+ if ((! myTdb().getTargetFile()) ||
+ (myTdb().getModTSforDir() == -1))
+ {
+ pstate.step_ = EXTRACT_INITIALIZE;
+ break;
+ }
+
+ numBuffers_ = 0;
+
+ memset (hdfsHost_, '\0', sizeof(hdfsHost_));
+ strncpy(hdfsHost_, myTdb().getHdfsHostName(), sizeof(hdfsHost_));
+ hdfsPort_ = myTdb().getHdfsPortNum();
+ memset (fileName_, '\0', sizeof(fileName_));
+ memset (targetLocation_, '\0', sizeof(targetLocation_));
+ snprintf(targetLocation_,999, "%s", myTdb().getTargetName());
+
+ retcode = lobInterfaceDataModCheck();
+ if (retcode < 0)
+ {
+ Lng32 cliError = 0;
+
+ Lng32 intParam1 = -retcode;
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
+ NULL, &intParam1,
+ &cliError,
+ NULL,
+ "HDFS",
+ (char*)"ExpLOBInterfaceDataModCheck",
+ getLobErrStr(intParam1));
+ pentry_down->setDiagsArea(diagsArea);
+ pstate.step_ = EXTRACT_ERROR;
+ break;
+ }
+
+ if (retcode == 1) // check failed
+ {
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(8436));
+ pentry_down->setDiagsArea(diagsArea);
+ pstate.step_ = EXTRACT_ERROR;
+ break;
+ }
+
pstate.step_= EXTRACT_INITIALIZE;
}
- // no break here
+ break;
+
case EXTRACT_INITIALIZE:
{
pstate.processingStarted_ = FALSE;
@@ -798,7 +862,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
break;
}
}
-
+
if (feStats)
{
feStats->setPartitionNumber(fileNum);
@@ -1123,13 +1187,16 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
}
else if (myTdb().getBypassLibhdfs())
{
- sfwRetCode = sequenceFileWriter_->hdfsClose();
- if (!errorOccurred_ && sfwRetCode != SFW_OK )
- {
- createSequenceFileError(sfwRetCode);
- pstate.step_ = EXTRACT_ERROR;
- break;
- }
+ if (sequenceFileWriter_)
+ {
+ sfwRetCode = sequenceFileWriter_->hdfsClose();
+ if (!errorOccurred_ && sfwRetCode != SFW_OK )
+ {
+ createSequenceFileError(sfwRetCode);
+ pstate.step_ = EXTRACT_ERROR;
+ break;
+ }
+ }
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/executor/ExFastTransport.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.h b/core/sql/executor/ExFastTransport.h
index 2df9fe9..5de05aa 100644
--- a/core/sql/executor/ExFastTransport.h
+++ b/core/sql/executor/ExFastTransport.h
@@ -255,6 +255,7 @@ public:
enum FastExtractStates
{
EXTRACT_NOT_STARTED = 0,
+ EXTRACT_CHECK_MOD_TS,
EXTRACT_INITIALIZE,
EXTRACT_PASS_REQUEST_TO_CHILD,
EXTRACT_RETURN_ROWS_FROM_CHILD,
@@ -366,6 +367,9 @@ protected:
time_t tstart_;
UInt32 bufferAllocFailuresCount_;
+
+ // modification timestamp of root dir location.
+ Int64 modTS_;
}; // class ExFastExtractTcb
/////////////////////////////////////////////////////
@@ -401,6 +405,7 @@ protected:
Lng32 lobInterfaceInsert(ssize_t bytesToWrite);
Lng32 lobInterfaceCreate();
Lng32 lobInterfaceClose();
+ Lng32 lobInterfaceDataModCheck();
virtual void insertUpQueueEntry(ex_queue::up_status status,
ComDiagsArea *diags,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 3ac537b..7db4a40 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -110,8 +110,7 @@ ExLob::~ExLob()
}
-__thread hdfsFS *globalFS = NULL;
-
+#ifdef __ignore
Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
char *dir,
LobsStorage storage,
@@ -147,20 +146,13 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
hdfsServer_ = hdfsServer;
hdfsPort_ = hdfsPort;
- if (globalFS == NULL)
- {
- globalFS = new hdfsFS;
- *globalFS = NULL;
- }
-
- if (*globalFS == NULL)
+ if (fs_ == NULL)
{
- *globalFS = hdfsConnect(hdfsServer_, hdfsPort_);
- if (*globalFS == NULL)
+ fs_ = hdfsConnect(hdfsServer_, hdfsPort_);
+ if (fs_ == NULL)
return LOB_HDFS_CONNECT_ERROR;
}
- fs_ = *globalFS;
if (lobGlobals)
lobGlobals->setHdfsFs(fs_);
@@ -214,6 +206,103 @@ Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
return LOB_OPER_OK;
}
+#endif
+
+Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
+ char *dir,
+ LobsStorage storage,
+ char *hdfsServer, Int64 hdfsPort,
+ char *lobLocation,
+ int bufferSize , short replication ,
+ int blockSize, Int64 lobMaxSize, ExLobGlobals *lobGlobals)
+{
+ int openFlags;
+ mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
+ struct timespec startTime;
+ struct timespec endTime;
+ Int64 secs, nsecs, totalnsecs;
+
+ if (dir)
+ {
+ if (dir_.empty())
+ {
+ dir_ = string(dir);
+ }
+
+ if (lobFile)
+ snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(),
+ lobFile);
+
+ }
+ else
+ {
+ if (lobFile)
+ snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile);
+
+ }
+
+ if (storage_ != Lob_Invalid_Storage)
+ {
+ return LOB_INIT_ERROR;
+ } else
+ {
+ storage_ = storage;
+ }
+
+ stats_.init();
+
+ hdfsServer_ = hdfsServer;
+ hdfsPort_ = hdfsPort;
+ lobLocation_ = lobLocation;
+ clock_gettime(CLOCK_MONOTONIC, &startTime);
+
+ if (lobGlobals->getHdfsFs() == NULL)
+ {
+ fs_ = hdfsConnect(hdfsServer_, hdfsPort_);
+ if (fs_ == NULL)
+ return LOB_HDFS_CONNECT_ERROR;
+ lobGlobals->setHdfsFs(fs_);
+ }
+ else
+ {
+ fs_ = lobGlobals->getHdfsFs();
+ }
+
+ clock_gettime(CLOCK_MONOTONIC, &endTime);
+
+ secs = endTime.tv_sec - startTime.tv_sec;
+ nsecs = endTime.tv_nsec - startTime.tv_nsec;
+ if (nsecs < 0)
+ {
+ secs--;
+ nsecs += NUM_NSECS_IN_SEC;
+ }
+ totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
+ stats_.hdfsConnectionTime += totalnsecs;
+
+ if (mode == EX_LOB_CREATE)
+ {
+ // check if file is already created
+ hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
+ if (fInfo != NULL)
+ {
+ hdfsFreeFileInfo(fInfo, 1);
+ return LOB_DATA_FILE_CREATE_ERROR;
+ }
+ openFlags = O_WRONLY | O_CREAT;
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize);
+ if (!fdData_)
+ {
+ return LOB_DATA_FILE_CREATE_ERROR;
+ }
+ hdfsCloseFile(fs_, fdData_);
+ fdData_ = NULL;
+
+ }
+ lobGlobalHeap_ = lobGlobals->getHeap();
+ return LOB_OPER_OK;
+
+}
Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD, Int64 transId)
{
@@ -443,13 +532,24 @@ Ex_Lob_Error ExLob::dataModCheck2(
Ex_Lob_Error ExLob::dataModCheck(
char * dirPath,
Int64 inputModTS,
- Lng32 numOfPartLevels)
+ Lng32 numOfPartLevels,
+ ExLobGlobals *lobGlobals)
{
// find mod time of root dir
hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, dirPath);
if (fileInfos == NULL)
{
- return LOB_DATA_FILE_NOT_FOUND_ERROR;
+ hdfsDisconnect(fs_);
+ fs_ = hdfsConnect(hdfsServer_, hdfsPort_);
+ if (fs_ == NULL)
+ return LOB_HDFS_CONNECT_ERROR;
+
+ fileInfos = hdfsGetPathInfo(fs_, dirPath);
+ if (fileInfos == NULL)
+ return LOB_DIR_NAME_ERROR;
+
+ if (lobGlobals)
+ lobGlobals->setHdfsFs(fs_);
}
Int64 currModTS = fileInfos[0].mLastMod;
@@ -471,12 +571,18 @@ Ex_Lob_Error ExLob::emptyDirectory()
Ex_Lob_Error err;
int numExistingFiles=0;
- hdfsFileInfo *fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles);
+ hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_);
if (fileInfos == NULL)
- {
- return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory
- }
+ {
+ return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory
+ }
+ fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles);
+ if (fileInfos == NULL)
+ {
+ return LOB_OPER_OK;
+ }
+
for (int i = 0; i < numExistingFiles; i++)
{
#ifdef USE_HADOOP_1
@@ -2167,7 +2273,8 @@ Ex_Lob_Error ExLobsOper (
if (globPtr == NULL)
{
- if (operation == Lob_Init)
+ if ((operation == Lob_Init) ||
+ (operation == Lob_Data_Mod_Check))
{
globPtr = (void *) new ExLobGlobals();
if (globPtr == NULL)
@@ -2176,14 +2283,16 @@ Ex_Lob_Error ExLobsOper (
lobGlobals = (ExLobGlobals *)globPtr;
err = lobGlobals->initialize();
- return err;
+ if (err != LOB_OPER_OK)
+ return err;
}
else
{
return LOB_GLOB_PTR_ERROR;
}
}
- else
+
+ if ((globPtr != NULL) && (operation != Lob_Init))
{
lobGlobals = (ExLobGlobals *)globPtr;
@@ -2236,6 +2345,7 @@ Ex_Lob_Error ExLobsOper (
*/
switch(operation)
{
+ case Lob_Init:
case Lob_Create:
break;
@@ -2425,7 +2535,8 @@ Ex_Lob_Error ExLobsOper (
Int64 inputModTS = *(Int64*)blackBox;
Int32 inputNumOfPartLevels =
*(Lng32*)&((char*)blackBox)[sizeof(inputModTS)];
- err = lobPtr->dataModCheck(dir, inputModTS, inputNumOfPartLevels);
+ err = lobPtr->dataModCheck(dir, inputModTS, inputNumOfPartLevels,
+ lobGlobals);
}
break;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 518fbd7..416529d 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -489,7 +489,9 @@ class ExLob
Ex_Lob_Error dataModCheck(
char * dirPath,
Int64 modTS,
- Lng32 numOfPartLevels);
+ Lng32 numOfPartLevels,
+ ExLobGlobals *lobGlobals);
+
Ex_Lob_Error dataModCheck2(
char * dirPath,
Int64 modTS,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/generator/GenFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp
index 953176b..7e1ee3e 100644
--- a/core/sql/generator/GenFastTransport.cpp
+++ b/core/sql/generator/GenFastTransport.cpp
@@ -562,6 +562,18 @@ PhysicalFastExtract::codeGen(Generator *generator)
newRecordSep[1] = '\0';
}
+ Int64 modTS = -1;
+ if ((CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON) &&
+ (isHiveInsert()) &&
+ (getHiveTableDesc() && getHiveTableDesc()->getNATable() &&
+ getHiveTableDesc()->getNATable()->getClusteringIndex()))
+ {
+ const HHDFSTableStats* hTabStats =
+ getHiveTableDesc()->getNATable()->getClusteringIndex()->getHHDFSTableStats();
+
+ modTS = hTabStats->getModificationTS();
+ }
+
targetName = AllocStringInSpace(*space, (char *)getTargetName().data());
hdfsHostName = AllocStringInSpace(*space, (char *)getHdfsHostName().data());
hiveTableName = AllocStringInSpace(*space, (char *)getHiveTableName().data());
@@ -570,27 +582,27 @@ PhysicalFastExtract::codeGen(Generator *generator)
recordSeparator = AllocStringInSpace(*space, newRecordSep);
nullString = AllocStringInSpace(*space, (char *)getNullString().data());
- result = ft_codegen(generator,
- *this, // RelExpr &relExpr
- newTdb, // ComTdbUdr *&newTdb
- estimatedRowCount,
- targetName,
- hdfsHostName,
- hdfsPortNum,
- hiveTableName,
- delimiter,
- header,
- nullString,
- recordSeparator,
- downQueueMaxSize,
- upQueueMaxSize,
- outputBufferSize,
- requestBufferSize,
- replyBufferSize,
- numOutputBuffers,
- childTdb,
- isSequenceFile());
-
+ result = ft_codegen(generator,
+ *this, // RelExpr &relExpr
+ newTdb, // ComTdbUdr *&newTdb
+ estimatedRowCount,
+ targetName,
+ hdfsHostName,
+ hdfsPortNum,
+ hiveTableName,
+ delimiter,
+ header,
+ nullString,
+ recordSeparator,
+ downQueueMaxSize,
+ upQueueMaxSize,
+ outputBufferSize,
+ requestBufferSize,
+ replyBufferSize,
+ numOutputBuffers,
+ childTdb,
+ isSequenceFile());
+
if (!generator->explainDisabled())
{
generator->setExplainTuple(addExplainInfo(newTdb, firstExplainTuple, 0, generator));
@@ -626,8 +638,10 @@ PhysicalFastExtract::codeGen(Generator *generator)
else
GenAssert(0, "Unexpected Fast Extract compression type")
}
- if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON)
- newTdb->setPrintDiags(1);
+ if((ActiveSchemaDB()->getDefaults()).getToken(FAST_EXTRACT_DIAGS) == DF_ON)
+ newTdb->setPrintDiags(1);
+
+ newTdb->setModTSforDir(modTS);
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index 4d3f9b6..fda6611 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -1070,7 +1070,8 @@ void HHDFSTableStats::print(FILE *ofd)
fprintf(ofd,"====================================================================\n");
}
-extern __thread hdfsFS *globalFS;
+//extern __thread hdfsFS *globalFS;
+hdfsFS *globalFS;
NABoolean HHDFSTableStats::connectHDFS(const NAString &host, Int32 port)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003
index 46aede2..79cdb5a 100644
--- a/core/sql/regress/hive/EXPECTED003
+++ b/core/sql/regress/hive/EXPECTED003
@@ -5,9 +5,6 @@
>>cqd attempt_esp_parallelism 'off';
--- SQL operation complete.
->>cqd AUTO_QUERY_RETRY 'OFF';
-
---- SQL operation complete.
>>cqd hive_max_esps '1';
--- SQL operation complete.
@@ -152,9 +149,6 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
>>
>>--try new HIVE SYNTAX
>>--------------
->>cqd query_cache '0';
-
---- SQL operation complete.
>>insert into TABLE hive.ins_customer_address select * from hive.customer_address;
--- 50000 row(s) inserted.
@@ -529,9 +523,6 @@ T_TIME_SK T_TIME_ID T_TIME T_HOUR T_MINUTE T
>>cqd HIVE_MAX_STRING_LENGTH '25' ;
--- SQL operation complete.
->>cqd query_cache '0';
-
---- SQL operation complete.
>>control query shape esp_exchange(cut);
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index 3c286d6..8e26184 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -226,9 +226,9 @@ xyz
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
- 19631
+ 1384
+N 19631
Y 18984
-? 1384
--- 3 row(s) selected.
>>execute s3;
@@ -316,9 +316,9 @@ A B
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
- 19631
+ 1384
+N 19631
Y 18984
-? 1384
--- 3 row(s) selected.
>>execute s4;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED006
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED006 b/core/sql/regress/hive/EXPECTED006
index 645cca9..87f630e 100644
--- a/core/sql/regress/hive/EXPECTED006
+++ b/core/sql/regress/hive/EXPECTED006
@@ -4,9 +4,6 @@
--- SQL operation complete.
>>set terminal_charset utf8;
>>
->>cqd AUTO_QUERY_RETRY 'OFF';
-
---- SQL operation complete.
>>cqd HIVE_MAX_STRING_LENGTH '25' ;
--- SQL operation complete.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index 01f4e0b..2c34c31 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -150,9 +150,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Rows Processed: 50000
-Task: PREPARATION Status: Ended ET: 00:00:09.173
+Task: PREPARATION Status: Ended ET: 00:00:06.864
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: COMPLETION Status: Ended ET: 00:00:00.304
+Task: COMPLETION Status: Ended ET: 00:00:00.262
--- 50000 row(s) loaded.
>>--
@@ -181,9 +181,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:11.448
+Task: PREPARATION Status: Ended ET: 00:00:10.758
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:00.275
+Task: COMPLETION Status: Ended ET: 00:00:00.234
--- 20000 row(s) loaded.
>>--
@@ -213,9 +213,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:07.319
+Task: PREPARATION Status: Ended ET: 00:00:09.668
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.198
+Task: COMPLETION Status: Ended ET: 00:00:00.208
--- 20000 row(s) loaded.
>>--
@@ -235,9 +235,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
Rows Processed: 100000
-Task: PREPARATION Status: Ended ET: 00:00:08.340
+Task: PREPARATION Status: Ended ET: 00:00:07.549
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.212
+Task: COMPLETION Status: Ended ET: 00:00:00.167
--- 100000 row(s) loaded.
>>--
@@ -266,9 +266,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SA
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
Rows Processed: 160756
-Task: PREPARATION Status: Ended ET: 00:00:10.675
+Task: PREPARATION Status: Ended ET: 00:00:11.382
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.256
+Task: COMPLETION Status: Ended ET: 00:00:00.180
--- 160756 row(s) loaded.
>>--
@@ -288,6 +288,10 @@ Task: COMPLETION Status: Ended ET: 00:00:00.256
>>-- using insert
>>insert overwrite table hive.hive.null_format_default select * from null_format_src;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 10 row(s) inserted.
>>select * from hive.hive.null_format_default;
@@ -309,6 +313,10 @@ a
>>
>>insert overwrite table hive.hive.null_format_empty select * from null_format_src;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 10 row(s) inserted.
>>select * from hive.hive.null_format_empty;
@@ -330,6 +338,10 @@ a ?
>>
>>insert overwrite table hive.hive.null_format_colon select * from null_format_src;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 10 row(s) inserted.
>>select * from hive.hive.null_format_colon;
@@ -356,10 +368,10 @@ a
+> select * from null_format_src;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.238
+Task: EXTRACT Status: Ended ET: 00:00:00.232
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_default;
@@ -385,10 +397,10 @@ a
+> select * from null_format_src;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.246
+Task: EXTRACT Status: Ended ET: 00:00:00.218
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_empty;
@@ -414,10 +426,10 @@ a ?
+> select * from null_format_src;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.009
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.222
+Task: EXTRACT Status: Ended ET: 00:00:00.213
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_colon;
@@ -471,12 +483,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.011
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.786
+Task: EXTRACT Status: Ended ET: 00:00:03.805
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.036
+Task: MERGE FILES Status: Ended ET: 00:00:00.034
--- 50000 row(s) unloaded.
>>log;
@@ -509,12 +521,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.868
+Task: EXTRACT Status: Ended ET: 00:00:00.865
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.031
+Task: MERGE FILES Status: Ended ET: 00:00:00.023
--- 20000 row(s) unloaded.
>>log;
@@ -533,12 +545,12 @@ cat /tmp/merged_customer_demogs | wc -l
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.837
+Task: EXTRACT Status: Ended ET: 00:00:00.766
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.029
+Task: MERGE FILES Status: Ended ET: 00:00:00.023
--- 20000 row(s) unloaded.
>>log;
@@ -570,12 +582,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.024
+Task: EXTRACT Status: Ended ET: 00:00:00.855
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.041
+Task: MERGE FILES Status: Ended ET: 00:00:00.032
--- 20000 row(s) unloaded.
>>
@@ -593,12 +605,12 @@ regrhadoop.ksh fs -du -s /bulkload/customer_demographics_salt/merged_customer_de
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.722
+Task: EXTRACT Status: Ended ET: 00:00:00.692
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.048
+Task: MERGE FILES Status: Ended ET: 00:00:00.031
--- 20000 row(s) unloaded.
>>
@@ -631,10 +643,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.013
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.591
+Task: EXTRACT Status: Ended ET: 00:00:00.484
--- 20000 row(s) unloaded.
>>
@@ -654,12 +666,12 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file |
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.012
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.672
+Task: EXTRACT Status: Ended ET: 00:00:00.580
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.040
+Task: MERGE FILES Status: Ended ET: 00:00:00.038
--- 20000 row(s) unloaded.
>>
@@ -792,12 +804,12 @@ CD_DEMO_SK CD_GENDER
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.616
+Task: EXTRACT Status: Ended ET: 00:00:00.487
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.055
+Task: MERGE FILES Status: Ended ET: 00:00:00.030
--- 20000 row(s) unloaded.
>>log;
@@ -835,9 +847,9 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.562
+Task: EXTRACT Status: Ended ET: 00:00:00.492
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.046
+Task: MERGE FILES Status: Ended ET: 00:00:00.033
--- 20000 row(s) unloaded.
>>
@@ -887,15 +899,19 @@ CD_DEMO_SK CD_GENDER
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.480
+Task: EXTRACT Status: Ended ET: 00:00:00.980
--- 20000 row(s) unloaded.
>>
>>select count(*) from hive.hive.unload_customer_demographics;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
(EXPR)
--------------------
@@ -937,10 +953,10 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:01.946
+Task: EXTRACT Status: Ended ET: 00:00:02.111
--- 50000 row(s) unloaded.
>>
@@ -990,15 +1006,19 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.003
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:01.891
+Task: EXTRACT Status: Ended ET: 00:00:02.075
--- 50000 row(s) unloaded.
>>
>>select count(*) from hive.hive.unload_customer_address;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
(EXPR)
--------------------
@@ -1052,10 +1072,10 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>select * from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:06.124
+Task: EXTRACT Status: Ended ET: 00:00:06.402
--- 100000 row(s) unloaded.
>>select count(*) from hive.hive.unload_customer;
@@ -1105,10 +1125,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>select * from trafodion.hbase.customer_demographics_salt;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.144
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.930
+Task: EXTRACT Status: Ended ET: 00:00:00.985
--- 20000 row(s) unloaded.
>>
@@ -1158,12 +1178,12 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_address where ca_address_sk < 100;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 99
-Task: EXTRACT Status: Ended ET: 00:00:00.207
+Task: EXTRACT Status: Ended ET: 00:00:00.203
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.029
+Task: MERGE FILES Status: Ended ET: 00:00:00.021
--- 99 row(s) unloaded.
>>
@@ -1199,10 +1219,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/*
+>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 12349
-Task: EXTRACT Status: Ended ET: 00:00:06.278
+Task: EXTRACT Status: Ended ET: 00:00:06.351
--- 12349 row(s) unloaded.
>>
@@ -1321,10 +1341,10 @@ SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:07.279
+Task: EXTRACT Status: Ended ET: 00:00:07.648
--- 100000 row(s) unloaded.
>>
@@ -1372,10 +1392,10 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.141
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
Task: EXTRACT Status: Started
Rows Processed: 1998
-Task: EXTRACT Status: Ended ET: 00:00:00.957
+Task: EXTRACT Status: Ended ET: 00:00:00.490
--- 1998 row(s) unloaded.
>>
@@ -1486,7 +1506,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111
- snapshot_temp_location /bulkload/20160526204449/
+ snapshot_temp_location /bulkload/20160530055515/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1566,7 +1586,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20160526204456/
+ snapshot_temp_location /bulkload/20160530055522/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1648,11 +1668,11 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111
- snapshot_temp_location /bulkload/20160526204518/
+ snapshot_temp_location /bulkload/20160530055545/
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20160526204518/
+ snapshot_temp_location /bulkload/20160530055545/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1765,13 +1785,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: VERIFY SNAPSHO Status: Started
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.288
+Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.294
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.176
+Task: EXTRACT Status: Ended ET: 00:00:02.144
--- 50000 row(s) unloaded.
>>
@@ -1843,13 +1863,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.002
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.003
Task: VERIFY SNAPSHO Status: Started
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.337
+Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.311
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.975
+Task: EXTRACT Status: Ended ET: 00:00:02.081
--- 20000 row(s) unloaded.
>>
@@ -1900,18 +1920,22 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.474
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.416
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.272
+Task: EXTRACT Status: Ended ET: 00:00:01.191
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.014
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.010
--- 20000 row(s) unloaded.
>>
>>select count(*) from hive.hive.unload_customer_demographics;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
(EXPR)
--------------------
@@ -1954,16 +1978,16 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.020
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.009
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.597
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.602
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.095
+Task: EXTRACT Status: Ended ET: 00:00:01.481
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005
--- 20000 row(s) unloaded.
>>
@@ -1977,6 +2001,10 @@ Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
--- 1 row(s) selected.
>>select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk < 100 order by cd_demo_sk;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_ESTIMATE CD_CREDIT_RATING CD_DEP_COUNT CD_DEP_EMPLOYED_COUNT CD_DEP_COLLEGE_COUNT
----------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -------------------- ---------------------------------------------------------------------------------------------------- ------------ --------------------- --------------------
@@ -2012,16 +2040,16 @@ CD_DEMO_SK CD_GENDER
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.048
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.627
Task: EXTRACT Status: Started
Rows Processed: 1998
-Task: EXTRACT Status: Ended ET: 00:00:00.662
+Task: EXTRACT Status: Ended ET: 00:00:00.648
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.003
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004
--- 1998 row(s) unloaded.
>>
@@ -2096,13 +2124,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: CREATE SNAPSHO Status: Started
Snapshots created: 2
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.782
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.377
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:08.942
+Task: EXTRACT Status: Ended ET: 00:00:08.169
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 2
Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.007
@@ -2173,16 +2201,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select c_first_name,c_last_name from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.365
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.420
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:01.183
+Task: EXTRACT Status: Ended ET: 00:00:01.144
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.003
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.004
--- 100000 row(s) unloaded.
>>
@@ -2255,7 +2283,7 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS;
Task: UNLOAD Status: Started
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:01.997
+Task: EXTRACT Status: Ended ET: 00:00:01.974
--- 50000 row(s) unloaded.
>>--unload 24 -- should give an error
@@ -2320,7 +2348,7 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/*
Task: UNLOAD Status: Started
Task: EXTRACT Status: Started
Rows Processed but NOT Written to Disk: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.709
+Task: EXTRACT Status: Ended ET: 00:00:00.690
--- 20000 row(s) unloaded.
>>select count(*) from hive.hive.unload_customer_demographics;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003
index ada9c87..7e3641b 100644
--- a/core/sql/regress/hive/TEST003
+++ b/core/sql/regress/hive/TEST003
@@ -67,7 +67,6 @@ log LOG003 clear;
set schema hive.hive;
cqd attempt_esp_parallelism 'off';
-cqd AUTO_QUERY_RETRY 'OFF';
cqd hive_max_esps '1';
cqd PARALLEL_NUM_ESPS '1';
cqd HIVE_MAX_STRING_LENGTH '25' ;
@@ -101,7 +100,6 @@ insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f');
--try new HIVE SYNTAX
--------------
-cqd query_cache '0';
insert into TABLE hive.ins_customer_address select * from hive.customer_address;
select count(*) from hive.customer_address;
@@ -168,7 +166,6 @@ cqd PARALLEL_NUM_ESPS '2';
set schema hive;
cqd hive_max_esps '2';
cqd HIVE_MAX_STRING_LENGTH '25' ;
-cqd query_cache '0';
control query shape esp_exchange(cut);
prepare s from insert into table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk;
explain options 'f' s;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST006
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST006 b/core/sql/regress/hive/TEST006
index 10e3d55..37a2c8b 100644
--- a/core/sql/regress/hive/TEST006
+++ b/core/sql/regress/hive/TEST006
@@ -33,7 +33,6 @@ log LOG006 clear;
set schema hive.hive;
set terminal_charset utf8;
-cqd AUTO_QUERY_RETRY 'OFF';
cqd HIVE_MAX_STRING_LENGTH '25' ;
cqd HIST_ROWCOUNT_REQUIRING_STATS '50000';
cqd mode_seahive 'ON';
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/b1a8f024/core/sql/regress/hive/TEST018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018
index 034a2fb..86758e6 100644
--- a/core/sql/regress/hive/TEST018
+++ b/core/sql/regress/hive/TEST018
@@ -27,13 +27,11 @@ set schema trafodion.hbase;
cqd comp_bool_226 'on';
cqd TRAF_TABLE_SNAPSHOT_SCAN_TABLE_SIZE_THRESHOLD '0';
cqd hive_max_string_length '60';
-cqd query_cache '0';
cqd HIST_ROWCOUNT_REQUIRING_STATS '50000';
+cqd AUTO_QUERY_RETRY_WARNINGS 'ON';
obey TEST018(clean_up);
-
-
log LOG018 clear;
sh regrhive.ksh -v -f $REGRTSTDIR/TEST018_create_hive_tables.hive &> $REGRRUNDIR/LOG018_create_hive_tables.log ;