You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by an...@apache.org on 2016/05/31 22:53:42 UTC
[7/8] incubator-trafodion git commit: hive data modification
detection: commit #4
hive data modification detection: commit #4
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/51a9c73e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/51a9c73e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/51a9c73e
Branch: refs/heads/master
Commit: 51a9c73ed37c3711178483c4a200f17a92d1c08d
Parents: b1a8f02
Author: Anoop Sharma <an...@esgyn.com>
Authored: Tue May 31 18:03:40 2016 +0000
Committer: Anoop Sharma <an...@esgyn.com>
Committed: Tue May 31 18:03:40 2016 +0000
----------------------------------------------------------------------
core/sql/cli/SessionDefaults.cpp | 2 +
core/sql/comexe/ComTdbExeUtil.cpp | 6 +-
core/sql/comexe/ComTdbExeUtil.h | 12 ++-
core/sql/comexe/ComTdbHdfsScan.h | 4 +-
core/sql/executor/ExExeUtil.h | 9 +-
core/sql/executor/ExExeUtilMisc.cpp | 95 ++++++++++++++-----
core/sql/executor/ExFastTransport.cpp | 2 +-
core/sql/executor/ExHdfsScan.cpp | 2 +-
core/sql/exp/ExpErrorEnums.h | 1 +
core/sql/exp/ExpLOBaccess.cpp | 133 ++++-----------------------
core/sql/exp/ExpLOBaccess.h | 5 +-
core/sql/exp/ExpLOBinterface.h | 5 +-
core/sql/generator/GenFastTransport.cpp | 2 +-
core/sql/generator/GenRelExeUtil.cpp | 5 +-
core/sql/optimizer/BindRelExpr.cpp | 3 +-
core/sql/optimizer/HDFSHook.cpp | 50 ++++------
core/sql/optimizer/NATable.cpp | 3 +
core/sql/optimizer/RelExeUtil.h | 9 +-
core/sql/optimizer/RelFastTransport.cpp | 1 +
core/sql/regress/hive/EXPECTED003 | 37 ++++++--
core/sql/regress/hive/EXPECTED005 | 36 +++++---
core/sql/regress/hive/EXPECTED015 | 59 ++++++------
core/sql/regress/hive/TEST003 | 9 +-
core/sql/regress/hive/TEST005 | 11 +--
core/sql/regress/hive/TEST015 | 3 +-
25 files changed, 240 insertions(+), 264 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/cli/SessionDefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/SessionDefaults.cpp b/core/sql/cli/SessionDefaults.cpp
index 024754b..5e79138 100644
--- a/core/sql/cli/SessionDefaults.cpp
+++ b/core/sql/cli/SessionDefaults.cpp
@@ -782,6 +782,8 @@ static const AQRInfo::AQRErrorMap aqrErrorMap[] =
// parallel purgedata failed
AQREntry( 8022, 0, 3, 60, 0, 0, "", 0, 1),
+ // hive data modification timestamp mismatch.
+ // query will be AQR'd and hive metadata will be reloaded.
AQREntry( 8436, 0, 1, 0, 0, 2, "04:05", 0, 0),
// FS memory errors
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp
index 3442714..e3dd630 100644
--- a/core/sql/comexe/ComTdbExeUtil.cpp
+++ b/core/sql/comexe/ComTdbExeUtil.cpp
@@ -1189,7 +1189,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete(
NABoolean isHiveTruncate,
char * hiveTableLocation,
char * hiveHostName,
- Lng32 hivePortNum)
+ Lng32 hivePortNum,
+ Int64 hiveModTS)
: ComTdbExeUtil(ComTdbExeUtil::FAST_DELETE_,
NULL, 0, (Int16)SQLCHARSETCODE_UNKNOWN,
tableName, tableNameLen,
@@ -1211,7 +1212,8 @@ ComTdbExeUtilFastDelete::ComTdbExeUtilFastDelete(
lobNumArray_(lobNumArray),
hiveTableLocation_(hiveTableLocation),
hiveHdfsHost_(hiveHostName),
- hiveHdfsPort_(hivePortNum)
+ hiveHdfsPort_(hivePortNum),
+ hiveModTS_(hiveModTS)
{
setIsHiveTruncate(isHiveTruncate);
setNodeType(ComTdb::ex_FAST_DELETE);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index ea3ef69..eb70eac 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -1518,7 +1518,8 @@ public:
NABoolean ishiveTruncate = FALSE,
char * hiveTableLocation = NULL,
char * hiveHostName = NULL,
- Lng32 hivePortNum = 0
+ Lng32 hivePortNum = 0,
+ Int64 hiveModTS = -1
);
Long pack(void *);
@@ -1569,6 +1570,11 @@ public:
return hiveHdfsPort_;
}
+ Lng32 getHiveModTS() const
+ {
+ return hiveModTS_;
+ }
+
// ---------------------------------------------------------------------
// Used by the internal SHOWPLAN command to get attributes of a TDB.
// ---------------------------------------------------------------------
@@ -1645,7 +1651,9 @@ private:
NABasicPtr hiveTableLocation_; // 56-63
NABasicPtr hiveHdfsHost_; // 64-71
Int32 hiveHdfsPort_; // 72-75
- char fillersComTdbExeUtilFastDelete_[52]; // 76-127
+ char fillers1_[4]; // 76-79
+ Int64 hiveModTS_; // 80-87
+ char fillersComTdbExeUtilFastDelete_[40]; // 88-127
};
class ComTdbExeUtilGetStatistics : public ComTdbExeUtil
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/comexe/ComTdbHdfsScan.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHdfsScan.h b/core/sql/comexe/ComTdbHdfsScan.h
index ac83311..392f7cf 100755
--- a/core/sql/comexe/ComTdbHdfsScan.h
+++ b/core/sql/comexe/ComTdbHdfsScan.h
@@ -135,7 +135,7 @@ class ComTdbHdfsScan : public ComTdb
char fillersComTdbHdfsScan1_[2]; // 190 - 191
NABasicPtr nullFormat_; // 192 - 199
- // next 3 params used to check if data under hdfsFileDir
+ // next 4 params are used to check if data under hdfsFileDir
// was modified after query was compiled.
NABasicPtr hdfsRootDir_; // 200 - 207
Int64 modTSforDir_; // 208 - 215
@@ -196,7 +196,7 @@ public:
char * loggingLocation = NULL,
char * errCountId = NULL,
- // next 3 params used to check if data under hdfsFileDir
+ // next 4 params are used to check if data under hdfsFileDir
// was modified after query was compiled.
char * hdfsRootDir = NULL,
Int64 modTSforDir = -1,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 6ed29c1..062362b 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -3299,6 +3299,7 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb
{
INITIAL_,
ERROR_,
+ DATA_MOD_CHECK_,
EMPTY_DIRECTORY_,
DONE_
};
@@ -3306,18 +3307,10 @@ class ExExeUtilHiveTruncateTcb : public ExExeUtilTcb
ExExeUtilFastDeleteTdb & fdTdb() const
{return (ExExeUtilFastDeleteTdb &) tdb;};
-
-// short doHiveTruncate(char * objectName,
-// NABoolean isIndex,
-// NABoolean fastDelUsingResetEOF);
-
short injectError(const char * val);
Step step_;
- char hdfsHost_[500];
- int hdfsPort_;
- char hiveTableLocation_[513];
int numExistingFiles_;
void * lobGlob_;
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExExeUtilMisc.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilMisc.cpp b/core/sql/executor/ExExeUtilMisc.cpp
index 1aea6c1..267d22e 100644
--- a/core/sql/executor/ExExeUtilMisc.cpp
+++ b/core/sql/executor/ExExeUtilMisc.cpp
@@ -2240,13 +2240,7 @@ ExExeUtilHiveTruncateTcb::ExExeUtilHiveTruncateTcb(
qparent_.down->allocatePstate(this);
numExistingFiles_ = 0;
- memset (hdfsHost_, '\0', sizeof(hdfsHost_));
- memset (hiveTableLocation_, '\0', sizeof(hiveTableLocation_));
- strncpy(hdfsHost_, fdTdb().getHiveHdfsHost(), sizeof(hdfsHost_));
- hdfsPort_ = fdTdb().getHiveHdfsPort();
- char * outputPath = fdTdb().getHiveTableLocation();
- strncpy(hiveTableLocation_, outputPath, 512);
step_ = INITIAL_;
}
@@ -2290,26 +2284,84 @@ short ExExeUtilHiveTruncateTcb::work()
case INITIAL_:
{
- //nothing for now --
- // more stuff later
+ if (fdTdb().getHiveModTS() > 0)
+ step_ = DATA_MOD_CHECK_;
+ else
+ step_ = EMPTY_DIRECTORY_;
+ }
+ break;
+
+ case DATA_MOD_CHECK_:
+ {
+ cliRC = ExpLOBinterfaceDataModCheck
+ (lobGlob_,
+ fdTdb().getHiveTableLocation(),
+ fdTdb().getHiveHdfsHost(),
+ fdTdb().getHiveHdfsPort(),
+ fdTdb().getHiveModTS(),
+ 0);
+
+ if (cliRC < 0)
+ {
+ Lng32 cliError = 0;
+
+ Lng32 intParam1 = -cliRC;
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
+ NULL, &intParam1,
+ &cliError,
+ NULL,
+ "HDFS",
+ (char*)"ExpLOBInterfaceEmptyDirectory",
+ getLobErrStr(intParam1));
+ pentry_down->setDiagsArea(diagsArea);
+ step_ = ERROR_;
+ break;
+ }
+
+ if (cliRC == 1) // data mod check failed
+ {
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
+ pentry_down->setDiagsArea(diagsArea);
+
+ step_ = ERROR_;
+ break;
+ }
+
step_ = EMPTY_DIRECTORY_;
}
- break;
+ break;
case EMPTY_DIRECTORY_:
{
- Lng32 retCode= ExpLOBinterfaceEmptyDirectory(
- lobGlob_,
- (char*)"", //name is empty
- hiveTableLocation_,
- Lob_HDFS_File,
- hdfsHost_,
- hdfsPort_,
- 0 ,
- 1 ,
- 0);
- if (retCode != 0)
+ cliRC = ExpLOBinterfaceEmptyDirectory(
+ lobGlob_,
+ (char*)"", //name is empty
+ fdTdb().getHiveTableLocation(),
+ Lob_HDFS_File,
+ fdTdb().getHiveHdfsHost(),
+ fdTdb().getHiveHdfsPort(),
+ 0 ,
+ 1 ,
+ 0);
+ if (cliRC != 0)
{
+ Lng32 cliError = 0;
+
+ Lng32 intParam1 = -cliRC;
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
+ NULL, &intParam1,
+ &cliError,
+ NULL,
+ "HDFS",
+ (char*)"ExpLOBInterfaceEmptyDirectory",
+ getLobErrStr(intParam1));
+ pentry_down->setDiagsArea(diagsArea);
step_ = ERROR_;
}
else
@@ -2318,6 +2370,7 @@ short ExExeUtilHiveTruncateTcb::work()
}
}
break;
+
case ERROR_:
{
if (qparent_.up->isFull())
@@ -2350,7 +2403,7 @@ short ExExeUtilHiveTruncateTcb::work()
step_ = DONE_;
}
- break;
+ break;
case DONE_:
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExFastTransport.cpp b/core/sql/executor/ExFastTransport.cpp
index 45c3959..bb8d4dc 100644
--- a/core/sql/executor/ExFastTransport.cpp
+++ b/core/sql/executor/ExFastTransport.cpp
@@ -738,7 +738,7 @@ ExWorkProcRetcode ExHdfsFastExtractTcb::work()
{
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(8436));
+ (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
pentry_down->setDiagsArea(diagsArea);
pstate.step_ = EXTRACT_ERROR;
break;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index b204fb2..82e01b3 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -468,7 +468,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
{
ComDiagsArea * diagsArea = NULL;
ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(8436));
+ (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
pentry_down->setDiagsArea(diagsArea);
step_ = HANDLE_ERROR_AND_DONE;
break;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpErrorEnums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpErrorEnums.h b/core/sql/exp/ExpErrorEnums.h
index de4d00f..987fe7e 100644
--- a/core/sql/exp/ExpErrorEnums.h
+++ b/core/sql/exp/ExpErrorEnums.h
@@ -153,6 +153,7 @@ enum ExeErrorCode
EXE_IS_BITWISE_AND_ERROR = 8431,
EXE_UNSIGNED_OVERFLOW = 8432,
EXE_INVALID_CHARACTER = 8433,
+ EXE_HIVE_DATA_MOD_CHECK_ERROR = 8436,
EXE_HISTORY_BUFFER_TOO_SMALL = 8440,
EXE_OLAP_OVERFLOW_NOT_SUPPORTED = 8441,
EXE_ERROR_FROM_LOB_INTERFACE = 8442,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 7db4a40..ad127ea 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -110,104 +110,6 @@ ExLob::~ExLob()
}
-#ifdef __ignore
-Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
- char *dir,
- LobsStorage storage,
- char *hdfsServer, Int64 hdfsPort,
- char *lobLocation,
- int bufferSize , short replication ,
- int blockSize, Int64 lobMaxSize,
- ExLobGlobals *lobGlobals)
-{
- int openFlags;
- mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
- struct timespec startTime;
- struct timespec endTime;
- Int64 secs, nsecs, totalnsecs;
-
- if (dir)
- {
- if (dir_.empty())
- {
- dir_ = string(dir);
- }
-
- if (lobFile)
- snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s/%s", dir_.c_str(), lobFile);
-
- }
- else if (lobFile)
- {
- snprintf(lobDataFile_, MAX_LOB_FILE_NAME_LEN, "%s", lobFile);
-
- }
-
- hdfsServer_ = hdfsServer;
- hdfsPort_ = hdfsPort;
-
- if (fs_ == NULL)
- {
- fs_ = hdfsConnect(hdfsServer_, hdfsPort_);
- if (fs_ == NULL)
- return LOB_HDFS_CONNECT_ERROR;
- }
-
- if (lobGlobals)
- lobGlobals->setHdfsFs(fs_);
-
- if (storage_ != Lob_Invalid_Storage)
- {
- return LOB_INIT_ERROR;
- }
- else
- {
- storage_ = storage;
- }
-
- stats_.init();
-
- if (lobLocation)
- lobLocation_ = lobLocation;
- clock_gettime(CLOCK_MONOTONIC, &startTime);
-
- clock_gettime(CLOCK_MONOTONIC, &endTime);
-
- secs = endTime.tv_sec - startTime.tv_sec;
- nsecs = endTime.tv_nsec - startTime.tv_nsec;
- if (nsecs < 0)
- {
- secs--;
- nsecs += NUM_NSECS_IN_SEC;
- }
- totalnsecs = (secs * NUM_NSECS_IN_SEC) + nsecs;
- stats_.hdfsConnectionTime += totalnsecs;
-
- if (mode == EX_LOB_CREATE)
- {
- // check if file is already created
- hdfsFileInfo *fInfo = hdfsGetPathInfo(fs_, lobDataFile_);
- if (fInfo != NULL)
- {
- hdfsFreeFileInfo(fInfo, 1);
- return LOB_DATA_FILE_CREATE_ERROR;
- }
- openFlags = O_WRONLY | O_CREAT;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags, bufferSize, replication, blockSize);
- if (!fdData_)
- {
- return LOB_DATA_FILE_CREATE_ERROR;
- }
- hdfsCloseFile(fs_, fdData_);
- fdData_ = NULL;
-
- }
- lobGlobalHeap_ = lobGlobals->getHeap();
- return LOB_OPER_OK;
-
-}
-#endif
-
Ex_Lob_Error ExLob::initialize(char *lobFile, Ex_Lob_Mode mode,
char *dir,
LobsStorage storage,
@@ -573,34 +475,34 @@ Ex_Lob_Error ExLob::emptyDirectory()
int numExistingFiles=0;
hdfsFileInfo *fileInfos = hdfsGetPathInfo(fs_, lobDataFile_);
if (fileInfos == NULL)
- {
- return LOB_DATA_FILE_NOT_FOUND_ERROR; //here a directory
- }
+ {
+ return LOB_DIR_NAME_ERROR;
+ }
fileInfos = hdfsListDirectory(fs_, lobDataFile_, &numExistingFiles);
- if (fileInfos == NULL)
- {
- return LOB_OPER_OK;
- }
-
- for (int i = 0; i < numExistingFiles; i++)
+ if (fileInfos == NULL) // empty directory
{
-#ifdef USE_HADOOP_1
- int retCode = hdfsDelete(fs_, fileInfos[i].mName);
-#else
- int retCode = hdfsDelete(fs_, fileInfos[i].mName, 0);
-#endif
+ return LOB_OPER_OK;
+ }
+
+ NABoolean error = FALSE;
+ for (int i = 0; ((NOT error) && (i < numExistingFiles)); i++)
+ {
+ // if dir, recursively delete it and everything under it
+ int retCode = hdfsDelete(fs_, fileInfos[i].mName, 1);
if (retCode !=0)
{
- //ex_assert(retCode == 0, "delete returned error");
- return LOB_DATA_FILE_DELETE_ERROR;
+ error = TRUE;
}
}
+
if (fileInfos)
{
hdfsFreeFileInfo(fileInfos, numExistingFiles);
}
-
+
+ if (error)
+ return LOB_DATA_FILE_DELETE_ERROR;
return LOB_OPER_OK;
}
@@ -2274,6 +2176,7 @@ Ex_Lob_Error ExLobsOper (
if (globPtr == NULL)
{
if ((operation == Lob_Init) ||
+ (operation == Lob_Empty_Directory) ||
(operation == Lob_Data_Mod_Check))
{
globPtr = (void *) new ExLobGlobals();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 416529d..2cce09e 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -481,10 +481,7 @@ class ExLob
// dirPath: path to needed directory (includes directory name)
// modTS is the latest timestamp on any file/dir under dirPath.
- // numFilesInDir is the total number of files under dirPath.
- // This method validates that current modTS is not greater then input modTS
- // and current number of files in dirPath are the same as input numFilesInDir.
- // If either condition is not true, then check fails.
+ // This method validates that current modTS is not greater then input modTS.
// Return: LOB_OPER_OK, if passes. LOB_DATA_MOD_CHECK_ERROR, if fails.
Ex_Lob_Error dataModCheck(
char * dirPath,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index 689e422..859417e 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -293,10 +293,7 @@ Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer,
// dirPath: path to needed directory (includes directory name)
// modTS is the latest timestamp on any file/dir under dirPath.
-// numFilesInDir is the total number of files under dirPath.
-// This method validates that current modTS is not greater then input modTS
-// and current number of files in dirPath are the same as input numFilesInDir.
-// If either condition is not true, then check fails.
+// This method validates that current modTS is not greater then input modTS.
// Return: 1, if check fails. 0, if passes. -1, if error.
Lng32 ExpLOBinterfaceDataModCheck(void * lobGlob,
char * dirPath,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenFastTransport.cpp b/core/sql/generator/GenFastTransport.cpp
index 7e1ee3e..6a4b4e7 100644
--- a/core/sql/generator/GenFastTransport.cpp
+++ b/core/sql/generator/GenFastTransport.cpp
@@ -624,7 +624,7 @@ PhysicalFastExtract::codeGen(Generator *generator)
{
newTdb->setIsHiveInsert(1);
newTdb->setIncludeHeader(0);
- setOverwriteHiveTable( getOverwriteHiveTable());
+ newTdb->setOverwriteHiveTable( getOverwriteHiveTable());
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index 55eba37..7629657 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -3237,12 +3237,11 @@ short ExeUtilFastDelete::codeGen(Generator * generator)
(ex_cri_desc *)(generator->getCriDesc(Generator::DOWN)),
(queue_index)getDefault(GEN_DDL_SIZE_DOWN),
(queue_index)getDefault(GEN_DDL_SIZE_UP),
-#pragma nowarn(1506) // warning elimination
getDefault(GEN_DDL_NUM_BUFFERS),
getDefault(GEN_DDL_BUFFER_SIZE),
isHiveTable(),
- hiveTableLocation, hiveHdfsHost, hiveHdfsPort);
-#pragma warn(1506) // warning elimination
+ hiveTableLocation, hiveHdfsHost, hiveHdfsPort,
+ hiveModTS_);
if (doPurgedataCat_)
exe_util_tdb->setDoPurgedataCat(TRUE);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index b81fdcf..9c3f0c2 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -9195,7 +9195,8 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
TRUE,
new (bindWA->wHeap()) NAString(tableDir),
new (bindWA->wHeap()) NAString(hostName),
- hdfsPort);
+ hdfsPort,
+ hTabStats->getModificationTS());
//new root to prevent error 4056 when binding
newRelExpr = new (bindWA->wHeap()) RelRoot(newRelExpr);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index fda6611..0d6fd34 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -1070,59 +1070,41 @@ void HHDFSTableStats::print(FILE *ofd)
fprintf(ofd,"====================================================================\n");
}
-//extern __thread hdfsFS *globalFS;
-hdfsFS *globalFS;
-
NABoolean HHDFSTableStats::connectHDFS(const NAString &host, Int32 port)
{
NABoolean result = TRUE;
// establish connection to HDFS if needed
- if (globalFS == NULL ||
- *globalFS == NULL ||
+ if (fs_ == NULL ||
currHdfsHost_ != host ||
currHdfsPort_ != port)
{
- if (globalFS && *globalFS)
- disconnectHDFS();
-
- if (globalFS == NULL)
+ if (fs_)
{
- globalFS = new hdfsFS;
- *globalFS = NULL;
+ hdfsDisconnect(fs_);
+ fs_ = NULL;
}
-
- if (*globalFS == NULL)
+ fs_ = hdfsConnect(host, port);
+
+ if (fs_ == NULL)
{
- *globalFS = hdfsConnect(host, port);
+ NAString errMsg("hdfsConnect to ");
+ errMsg += host;
+ errMsg += ":";
+ errMsg += port;
+ errMsg += " failed";
+ diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS");
+ result = FALSE;
}
-
currHdfsHost_ = host;
currHdfsPort_ = port;
}
-
- fs_ = *globalFS;
- if (fs_ == NULL)
- {
- NAString errMsg("hdfsConnect to ");
- errMsg += host;
- errMsg += ":";
- errMsg += port;
- errMsg += " failed";
- diags_.recordError(errMsg, "HHDFSTableStats::connectHDFS");
- result = FALSE;
- }
-
return result;
}
void HHDFSTableStats::disconnectHDFS()
{
- if (globalFS && *globalFS)
- {
- hdfsDisconnect(*globalFS);
- *globalFS = NULL;
- }
-
+ if (fs_)
+ hdfsDisconnect(fs_);
fs_ = NULL;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 81eb5b0..66d18b6 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -7308,6 +7308,9 @@ NATable * NATableDB::get(const ExtendedQualName* key, BindWA* bindWA, NABoolean
}
}
+ // the reload cqd will be set during aqr after compiletime and runtime
+ // timestamp mismatch is detected.
+ // If set, reload hive metadata.
if ((cachedNATable->isHiveTable()) &&
(CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON) &&
(CmpCommon::getDefault(TRAF_RELOAD_NATABLE_CACHE) == DF_ON))
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelExeUtil.h b/core/sql/optimizer/RelExeUtil.h
index 5d0150b..5c782e9 100644
--- a/core/sql/optimizer/RelExeUtil.h
+++ b/core/sql/optimizer/RelExeUtil.h
@@ -1034,7 +1034,8 @@ public:
NABoolean isHiveTable = FALSE,
NAString * hiveTableLocation = NULL,
NAString * hiveHostName = NULL,
- Int32 hiveHdfsPort = 0)
+ Int32 hiveHdfsPort = 0,
+ Int64 hiveModTS = -1)
: ExeUtilExpr(FAST_DELETE_, name, exprNode, NULL, stmtText, stmtTextCharSet, oHeap),
doPurgedataCat_(doPurgedataCat),
noLog_(noLog), ignoreTrigger_(ignoreTrigger),
@@ -1044,7 +1045,8 @@ public:
offlineTable_(FALSE),
doLabelPurgedata_(FALSE),
numLOBs_(0),
- isHiveTable_(isHiveTable)
+ isHiveTable_(isHiveTable),
+ hiveModTS_(hiveModTS)
{
if (isHiveTable )
{
@@ -1123,6 +1125,9 @@ private:
NAString hiveTableLocation_;
NAString hiveHostName_;
Int32 hiveHdfsPort_;
+
+ // timestamp of hiveTableLocation.
+ Int64 hiveModTS_;
};
class ExeUtilMaintainObject : public ExeUtilExpr
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/optimizer/RelFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp
index 45d4168..4734d2a 100644
--- a/core/sql/optimizer/RelFastTransport.cpp
+++ b/core/sql/optimizer/RelFastTransport.cpp
@@ -98,6 +98,7 @@ RelExpr * FastExtract::copyTopNode(RelExpr *derivedNode,
result->recordSeparator_ = recordSeparator_ ;
result->selectList_ = selectList_;
result->isSequenceFile_ = isSequenceFile_;
+ result->overwriteHiveTable_ = overwriteHiveTable_;
return RelExpr::copyTopNode(result, outHeap);
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003
index 79cdb5a..0d0c3d0 100644
--- a/core/sql/regress/hive/EXPECTED003
+++ b/core/sql/regress/hive/EXPECTED003
@@ -2,21 +2,18 @@
>>set schema hive.hive;
--- SQL operation complete.
->>cqd attempt_esp_parallelism 'off';
-
---- SQL operation complete.
->>cqd hive_max_esps '1';
-
---- SQL operation complete.
->>cqd PARALLEL_NUM_ESPS '1';
-
---- SQL operation complete.
+>>--cqd attempt_esp_parallelism 'off';
+>>--cqd hive_max_esps '1';
+>>--cqd PARALLEL_NUM_ESPS '1';
>>cqd HIVE_MAX_STRING_LENGTH '25' ;
--- SQL operation complete.
>>cqd mode_seahive 'ON';
--- SQL operation complete.
+>>cqd auto_query_retry_warnings 'ON';
+
+--- SQL operation complete.
>>
>>prepare s from insert into hive.ins_customer select * from hive.customer;
@@ -135,6 +132,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
*** ERROR[8822] The statement was not prepared.
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[4023] The degree of each row value constructor (20) must equal the degree of the target table column list (19).
+
>>-- number of columns doesn't match
>>
>>prepare s from
@@ -144,6 +145,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
*** ERROR[8822] The statement was not prepared.
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[4039] Column T_TIME_SK is of type INTEGER, incompatible with the value's type, CHAR(1).
+
>>-- wrong data types
>>
>>
@@ -172,6 +177,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
>>
>>insert OVERWRITE TABLE hive.ins_customer_address select * from hive.customer_address;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 50000 row(s) inserted.
>>
>>select count(*) from hive.customer_address;
@@ -193,6 +202,10 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
>>--execute again
>>insert OVERWRITE TABLE hive.ins_customer_address select * from hive.customer_address;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 50000 row(s) inserted.
>>
>>select count(*) from hive.customer_address;
@@ -595,7 +608,7 @@ test lp bug # 1355477
> SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY
> --------------- ----------- -----------
>>--execute again --overwrite should get rid og existing data from previous run
->>control query shape union(cut,esp_exchange(cut));
+>>control query shape union(cut, esp_exchange(cut));
--- SQL operation complete.
>>prepare s from insert overwrite table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk;
@@ -624,6 +637,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
--- SQL operation complete.
>>execute s;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
--- 12768 row(s) inserted.
>>control query shape cut;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index 8e26184..eaa414e 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -161,11 +161,12 @@ Y 9525
>>insert into newtable values ('abc');
--- 1 row(s) inserted.
->>cqd query_cache '0';
-
---- SQL operation complete.
>>select * from newtable;
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
A
-------------------------
@@ -173,9 +174,6 @@ abc
--- 1 row(s) selected.
>>-- expect to see the row, but only because query cache is off
->>cqd query_cache reset;
-
---- SQL operation complete.
>>
>>insert into hiveregr5.newtable2 values ('xyz');
@@ -289,9 +287,6 @@ A B
>>-- overwrite the table with auto-generated partitions
>>sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql;
>>
->>cqd query_cache '0';
-
---- SQL operation complete.
>>prepare s4 from
+> select c_preferred_cust_flag,
+> count(*)
@@ -652,14 +647,18 @@ C1 C2 C3 C4 C
>>sh echo "create table thive(a int);" > TEST005_junk;
>>sh regrhive.ksh -f TEST005_junk;
>>
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
--- 0 row(s) selected.
>>
>>sh echo "insert into thive values (1);" > TEST005_junk;
>>sh regrhive.ksh -f TEST005_junk;
>>
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
+
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
A
-----------
@@ -670,7 +669,7 @@ A
>>insert into hive.hive.thive values (2);
--- 1 row(s) inserted.
->>select * from hive.hive.thive;
+>>select a from hive.hive.thive;
A
-----------
@@ -689,6 +688,19 @@ A
>>sh echo "insert into thive values (1,2);" > TEST005_junk;
>>sh regrhive.ksh -f TEST005_junk;
>>
+>>select a from hive.hive.thive;
+
+*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
+
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions.
+
+A
+-----------
+
+ 1
+
+--- 1 row(s) selected.
+>>
>>select * from hive.hive.thive;
A B
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/EXPECTED015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED015 b/core/sql/regress/hive/EXPECTED015
index f1d6067..162a3bc 100644
--- a/core/sql/regress/hive/EXPECTED015
+++ b/core/sql/regress/hive/EXPECTED015
@@ -115,10 +115,11 @@
>>cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100';
--- SQL operation complete.
->>cqd query_cache '0';
+>>cqd HIVE_NUM_ESPS_PER_DATANODE '3';
--- SQL operation complete.
->>cqd HIVE_NUM_ESPS_PER_DATANODE '3';
+>>
+>>cqd auto_query_retry_warnings 'ON';
--- SQL operation complete.
>>
@@ -209,9 +210,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.184
+Task: PREPARATION Status: Ended ET: 00:00:00.051
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.359
+Task: COMPLETION Status: Ended ET: 00:00:00.781
--- 5 row(s) loaded.
>>
@@ -233,9 +234,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.176
+Task: PREPARATION Status: Ended ET: 00:00:00.162
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.203
+Task: COMPLETION Status: Ended ET: 00:00:00.183
--- 5 row(s) loaded.
>>
@@ -258,9 +259,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.195
+Task: PREPARATION Status: Ended ET: 00:00:00.064
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.318
+Task: COMPLETION Status: Ended ET: 00:00:00.327
--- 5 row(s) loaded.
>>
@@ -288,9 +289,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.179
+Task: PREPARATION Status: Ended ET: 00:00:00.164
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.380
+Task: COMPLETION Status: Ended ET: 00:00:00.268
--- 5 row(s) loaded.
>>
@@ -461,9 +462,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:08.673
+Task: PREPARATION Status: Ended ET: 00:00:08.885
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: COMPLETION Status: Ended ET: 00:00:00.402
+Task: COMPLETION Status: Ended ET: 00:00:01.379
--- 5000 row(s) loaded.
>>
@@ -543,9 +544,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:01.062
+Task: PREPARATION Status: Ended ET: 00:00:01.054
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task: COMPLETION Status: Ended ET: 00:00:00.294
+Task: COMPLETION Status: Ended ET: 00:00:00.820
--- 5000 row(s) loaded.
>>
@@ -644,9 +645,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:04.130
+Task: PREPARATION Status: Ended ET: 00:00:03.594
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:00.631
+Task: COMPLETION Status: Ended ET: 00:00:00.323
--- 5000 row(s) loaded.
>>
@@ -691,8 +692,6 @@ CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_EST
>>---------------------
>>select count(*) from hive.hive.customer_demographics where cd_demo_sk <= 5000;
-*** WARNING[6008] Statistics for column (CD_DEMO_SK) from table HIVE.HIVE.CUSTOMER_DEMOGRAPHICS were not available. As a result, the access path chosen might not be the best possible.
-
(EXPR)
--------------------
@@ -750,9 +749,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:06.200
+Task: PREPARATION Status: Ended ET: 00:00:05.596
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:01.735
+Task: COMPLETION Status: Ended ET: 00:00:01.203
--- 5000 row(s) loaded.
>>
@@ -1262,9 +1261,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:06.517
+Task: PREPARATION Status: Ended ET: 00:00:04.891
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:02.242
+Task: COMPLETION Status: Ended ET: 00:00:02.090
--- 5000 row(s) loaded.
>>
@@ -1305,11 +1304,11 @@ Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Rows Processed: 1000
-Task: PREPARATION Status: Ended ET: 00:00:04.908
+Task: PREPARATION Status: Ended ET: 00:00:04.588
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:01.357
+Task: COMPLETION Status: Ended ET: 00:00:01.075
Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: POPULATE INDEX Status: Ended ET: 00:00:12.289
+Task: POPULATE INDEX Status: Ended ET: 00:00:11.865
--- 1000 row(s) loaded.
>>
@@ -1354,11 +1353,11 @@ Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:05.592
+Task: PREPARATION Status: Ended ET: 00:00:05.812
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:02.018
+Task: COMPLETION Status: Ended ET: 00:00:02.106
Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: POPULATE INDEX Status: Ended ET: 00:00:14.563
+Task: POPULATE INDEX Status: Ended ET: 00:00:12.813
--- 5000 row(s) loaded.
>>
@@ -1532,9 +1531,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE."customer_addr
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE."customer_address_delim"
Task: PREPARATION Status: Started Object: TRAFODION.HBASE."customer_address_delim"
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:04.154
+Task: PREPARATION Status: Ended ET: 00:00:03.585
Task: COMPLETION Status: Started Object: TRAFODION.HBASE."customer_address_delim"
-Task: COMPLETION Status: Ended ET: 00:00:01.729
+Task: COMPLETION Status: Ended ET: 00:00:01.575
--- 5000 row(s) loaded.
>>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003
index 7e3641b..3761906 100644
--- a/core/sql/regress/hive/TEST003
+++ b/core/sql/regress/hive/TEST003
@@ -66,11 +66,12 @@ order by s desc;
log LOG003 clear;
set schema hive.hive;
-cqd attempt_esp_parallelism 'off';
-cqd hive_max_esps '1';
-cqd PARALLEL_NUM_ESPS '1';
+--cqd attempt_esp_parallelism 'off';
+--cqd hive_max_esps '1';
+--cqd PARALLEL_NUM_ESPS '1';
cqd HIVE_MAX_STRING_LENGTH '25' ;
cqd mode_seahive 'ON';
+cqd auto_query_retry_warnings 'ON';
prepare s from insert into hive.ins_customer select * from hive.customer;
execute s;
@@ -187,7 +188,7 @@ sh diff -b LOG003_ORIG_STORE_SALES_SUMMARY.DAT LOG003_INS_STORE_SALES_SUMMARY.DA
log LOG003;
--execute again --overwrite should get rid og existing data from previous run
-control query shape union(cut,esp_exchange(cut));
+control query shape union(cut, esp_exchange(cut));
prepare s from insert overwrite table ins_store_sales_summary select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales group by ss_sold_date_sk ,ss_store_sk;
explain options 'f' s;
execute s;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005
index e44b633..cd3fa56 100644
--- a/core/sql/regress/hive/TEST005
+++ b/core/sql/regress/hive/TEST005
@@ -144,10 +144,8 @@ where c_customer_sk between 20000 and 39999;
select * from newtable;
-- no rows, but should know the new table
insert into newtable values ('abc');
-cqd query_cache '0';
select * from newtable;
-- expect to see the row, but only because query cache is off
-cqd query_cache reset;
insert into hiveregr5.newtable2 values ('xyz');
select * from hiveregr5.newtable2;
@@ -190,7 +188,6 @@ select a,b from newtable;
-- overwrite the table with auto-generated partitions
sh regrhive.ksh -v -f $REGRTSTDIR/TEST005_d.hive.sql;
-cqd query_cache '0';
prepare s4 from
select c_preferred_cust_flag,
count(*)
@@ -292,14 +289,14 @@ sh regrhive.ksh -f TEST005_junk;
sh echo "create table thive(a int);" > TEST005_junk;
sh regrhive.ksh -f TEST005_junk;
-select * from hive.hive.thive;
+select a from hive.hive.thive;
sh echo "insert into thive values (1);" > TEST005_junk;
sh regrhive.ksh -f TEST005_junk;
-select * from hive.hive.thive;
+select a from hive.hive.thive;
insert into hive.hive.thive values (2);
-select * from hive.hive.thive;
+select a from hive.hive.thive;
sh echo "drop table thive;" > TEST005_junk;
sh regrhive.ksh -f TEST005_junk;
@@ -310,6 +307,8 @@ sh regrhive.ksh -f TEST005_junk;
sh echo "insert into thive values (1,2);" > TEST005_junk;
sh regrhive.ksh -f TEST005_junk;
+select a from hive.hive.thive;
+
select * from hive.hive.thive;
log;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/51a9c73e/core/sql/regress/hive/TEST015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST015 b/core/sql/regress/hive/TEST015
index ed0dd12..dfac48f 100644
--- a/core/sql/regress/hive/TEST015
+++ b/core/sql/regress/hive/TEST015
@@ -39,9 +39,10 @@ obey TEST015(setup);
cqd COMPRESSED_INTERNAL_FORMAT 'ON';
cqd COMPRESSED_INTERNAL_FORMAT_BMO 'ON';
cqd COMPRESSED_INTERNAL_FORMAT_DEFRAG_RATIO '100';
-cqd query_cache '0';
cqd HIVE_NUM_ESPS_PER_DATANODE '3';
+cqd auto_query_retry_warnings 'ON';
+
obey TEST015(test_bulk_load_simple);
log;