You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2016/07/14 00:22:04 UTC
incubator-trafodion git commit: Adding AQR retry for ENOENT and
EAGAIN which may be encountered after an hdfsOpen on a file that has gone
through a timestamp check. Squashed commit of the following:
Repository: incubator-trafodion
Updated Branches:
refs/heads/master ea2087410 -> 249dfefac
Adding AQR retry for ENOENT and EAGAIN which may be encountered after an hdfsOpen on a file that has gone through a timestamp check.
Squashed commit of the following:
commit 52551b6461e0b761aeb37958ecd8673ad29bec89
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Tue Jul 12 20:27:33 2016 +0000
Check for NULL pointer
commit a7bb285fb7874b314acb6c74cde5e0b6911d6930
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Tue Jul 12 16:12:17 2016 +0000
Rework to pass errno at the lowest layer as output param
commit abb236336ee68765fcf9c534fbeae3fe6af5cdb6
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Mon Jul 11 19:22:04 2016 +0000
Removed all retries and added AQR retry for missing hdfs file after timestamp check.
commit 2530c4ebb8274bea8394067d2d2212b5bc3c97ef
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Mon Jul 11 07:11:35 2016 +0000
timeout to 20
commit 825bd5be2b0db4006bf82a0a9348338f7fc66236
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Mon Jul 11 03:59:11 2016 +0000
increase sleep
commit e471a4655ca8efe469e5dd8ab14748660fa5cdec
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Sun Jul 10 19:00:03 2016 +0000
Unconditionally retrying on open errors.
commit b0402243a17094f7fec9899a6afc5c31c48cc38b
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Sun Jul 10 03:31:01 2016 +0000
Retrying for ENOENT too
commit cbb2175282095edb73b30e79a9728f689b80c435
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Sat Jul 9 23:14:31 2016 +0000
Rework and returning errno
commit dd99c24deec09add76342e583193f79646fc9661
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Sat Jul 9 05:05:43 2016 +0000
fixing build issue
commit 66941774b1087ce564932af0c30ce50f15d0da67
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Sat Jul 9 04:40:59 2016 +0000
Increasing retry and add hdfsClose to cleanup .
commit 1934f2b3b7b32ee4a0396d9e56457ffc9f2cb9cf
Author: Sandhya Sundaresan <sa...@apache.org>
Date: Fri Jul 8 21:59:00 2016 +0000
Adding retry logic when making hdfsOpenFile calls to read an hdfs file to handle potential EAGAIN errors. [TRAFODION-2097]
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/249dfefa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/249dfefa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/249dfefa
Branch: refs/heads/master
Commit: 249dfefac344fbb7db72d1131fae15a75c50058d
Parents: ea20874
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Jul 14 00:18:01 2016 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Jul 14 00:18:01 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExHdfsScan.cpp | 28 ++++-
core/sql/exp/ExpLOBaccess.cpp | 117 ++++++++++++--------
core/sql/exp/ExpLOBaccess.h | 2 +-
core/sql/exp/ExpLOBinterface.cpp | 5 +-
core/sql/exp/ExpLOBinterface.h | 3 +-
core/sql/regress/hive/EXPECTED018 | 194 ++++++++++++++++-----------------
core/sql/regress/hive/TEST018 | 24 ++--
7 files changed, 207 insertions(+), 166 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 3b32d69..ec9f3f8 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -377,6 +377,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
ContextCli *currContext = getGlobals()->castToExExeStmtGlobals()->getCliGlobals()->currContext();
hdfsFS hdfs = currContext->getHdfsServerConnection(hdfsScanTdb().hostName_,hdfsScanTdb().port_);
hdfsFileInfo *dirInfo = NULL;
+ Int32 hdfsErrorDetail = 0;//this is errno returned form underlying hdfsOpenFile call.
while (!qparent_.down->isEmpty())
{
ex_queue_entry *pentry_down = qparent_.down->getHeadEntry();
@@ -561,6 +562,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
}
else
{
+
openType = 2; // must open
retcode = ExpLOBInterfaceSelectCursor
(lobGlob_,
@@ -582,9 +584,21 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
bytesRead_,
NULL,
1, // open
- openType //
+ openType, //
+ &hdfsErrorDetail
);
+ if ((retcode < 0) &&
+ ((hdfsErrorDetail == ENOENT) || (hdfsErrorDetail == EAGAIN)))
+ {
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
+ pentry_down->setDiagsArea(diagsArea);
+ step_ = HANDLE_ERROR_AND_DONE;
+ break;
+ }
+
// preopen next range.
if ( (currRangeNum_ + 1) < (beginRangeNum_ + numRanges_) )
{
@@ -615,7 +629,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
bytesRead_,
NULL,
1,// open
- openType
+ openType,
+ &hdfsErrorDetail
);
hdfsFileName_ = hdfo_->fileName();
@@ -631,7 +646,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL,
&intParam1,
- &cliError,
+ &hdfsErrorDetail,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceSelectCursor/open",
@@ -702,7 +717,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
}
else
{
-
+ Int32 hdfsErrorDetail = 0;///this is the errno returned from the underlying hdfs call.
retcode = ExpLOBInterfaceSelectCursor
(lobGlob_,
hdfsFileName_,
@@ -723,6 +738,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
hdfsScanBuffer_ + trailingPrevRead_,
2, // read
0 // openType, not applicable for read
+ &hdfsErrorDetail
);
if (hdfsStats_)
@@ -737,7 +753,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL,
&intParam1,
- &cliError,
+ &hdfsErrorDetail,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceSelectCursor/read",
@@ -1247,7 +1263,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
ExRaiseSqlError(getHeap(), &diagsArea,
(ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE), NULL,
&intParam1,
- &cliError,
+ &errno,
NULL,
"HDFS",
(char*)"ExpLOBInterfaceSelectCursor/close",
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 67ea3be..575bb92 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -552,8 +552,9 @@ Ex_Lob_Error ExLob::statSourceFile(char *srcfile, Int64 &sourceEOF)
if (srcType == HDFS_FILE)
{
hdfsFile sourceFile = hdfsOpenFile(fs_,srcfile,O_RDONLY,0,0,0);
- if (!sourceFile)
- return LOB_SOURCE_FILE_OPEN_ERROR;
+ if (!sourceFile)
+ return LOB_SOURCE_FILE_OPEN_ERROR;
+
hdfsFileInfo *sourceFileInfo = hdfsGetPathInfo(fs_,srcfile);
// get EOD from source hdfs file.
if (sourceFileInfo)
@@ -669,11 +670,9 @@ Ex_Lob_Error ExLob::readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &si
int openFlags = O_RDONLY;
hdfsFile fdSrcFile = hdfsOpenFile(fs_,srcfile, openFlags,0,0,0);
- if (fdSrcFile == NULL) {
+ if (fdSrcFile == NULL)
return LOB_SOURCE_FILE_OPEN_ERROR;
- }
-
-
+
fileData = (char *) (getLobGlobalHeap())->allocateMemory(size);
if (fileData == (char *)-1) {
return LOB_SOURCE_DATA_ALLOC_ERROR;
@@ -1301,8 +1300,11 @@ Ex_Lob_Error ExLob::openCursor(char *handleIn, Int32 handleInLen,Int64 transId)
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, Int64 range, Int64 bufMaxSize,
- Int64 maxBytes, Int64 waited, ExLobGlobals *lobGlobals)
+Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type,
+ Int64 range, Int64 bufMaxSize,
+ Int64 maxBytes, Int64 waited,
+ ExLobGlobals *lobGlobals,
+ Int32 *hdfsDetailError)
{
Ex_Lob_Error err;
cursor_t cursor;
@@ -1359,22 +1361,27 @@ Ex_Lob_Error ExLob::openDataCursor(char *file, LobsCursorType type, Int64 range,
it = lobCursors_.find(string(file, strlen(file))); // to get the actual cursor object in the map
if (!fdData_ || (openFlags_ != O_RDONLY))
- {
- hdfsCloseFile(fs_, fdData_);
- fdData_ = NULL;
- openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- lobCursorLock_.unlock();
- return LOB_DATA_FILE_OPEN_ERROR;
+ {
+ hdfsCloseFile(fs_, fdData_);
+ fdData_ = NULL;
+ openFlags_ = O_RDONLY;
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+
+ if (!fdData_)
+ {
+ openFlags_ = -1;
+ if (hdfsDetailError)
+ *hdfsDetailError = errno;
+ lobCursorLock_.unlock();
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+
+ if (hdfsSeek(fs_, fdData_, (it->second).descOffset_) == -1)
+ {
+ lobCursorLock_.unlock();
+ return LOB_DATA_FILE_POSITION_ERROR;
+ }
}
- }
-
- if (hdfsSeek(fs_, fdData_, (it->second).descOffset_) == -1) {
- lobCursorLock_.unlock();
- return LOB_DATA_FILE_POSITION_ERROR;
- }
// start reading in a worker thread
lobGlobals->enqueuePrefetchRequest(this, &(it->second));
@@ -1552,15 +1559,17 @@ Ex_Lob_Error ExLob::compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int
hdfsFile fdData = hdfsOpenFile(fs, lobDataFile_, O_RDONLY, 0, 0,0);
- if (!fdData)
- {
+
+ if (!fdData)
+ {
str_sprintf(logBuf,"Could not open file:%s",lobDataFile_);
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
hdfsCloseFile(fs,fdData);
fdData = NULL;
return LOB_DATA_FILE_OPEN_ERROR;
}
-
+
+
hdfsFile fdTemp = hdfsOpenFile(fs, tmpLobDataFile,O_WRONLY|O_CREAT,0,0,0);
if (!fdTemp)
{
@@ -1768,16 +1777,20 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
// #endif
if (!fdData_ || (openFlags_ != O_RDONLY))
- {
- hdfsCloseFile(fs_, fdData_);
- fdData_=NULL;
- openFlags_ = O_RDONLY;
- fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
- }
- }
+ {
+ hdfsCloseFile(fs_, fdData_);
+ fdData_=NULL;
+ openFlags_ = O_RDONLY;
+ fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
+
+ if (!fdData_)
+ {
+ openFlags_ = -1;
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+
+
+ }
clock_gettime(CLOCK_MONOTONIC, &startTime);
@@ -1844,20 +1857,30 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
fdData_=NULL;
openFlags_ = O_RDONLY;
fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
- }
+
+ if (!fdData_)
+ {
+ openFlags_ = -1;
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+
+
}
else
{
fdData_ = hdfsOpenFile(fs_, lobDataFile_, openFlags_, 0, 0, 0);
- if (!fdData_) {
- openFlags_ = -1;
- return LOB_DATA_FILE_OPEN_ERROR;
- }
+
+ if (!fdData_)
+ {
+ openFlags_ = -1;
+ return LOB_DATA_FILE_OPEN_ERROR;
+ }
+
+
}
-
+
+
+
if (!multipleChunks)
{
lobDebugInfo("Reading in single chunk",0,__LINE__,lobTrace_);
@@ -2361,7 +2384,7 @@ Ex_Lob_Error ExLobsOper (
} else if (openType == 2) { // must open
sprintf(fn,"%s:%Lx:%s",lobPtr->getDataFileName(), (long long unsigned int)lobName, cursorId);
fileName = fn;
- err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals);
+ err = lobPtr->openDataCursor(fileName, Lob_Cursor_Simple, descNumIn, sourceLen, cursorBytes, waited, lobGlobals, (Int32 *)blackBox);
} else
err = LOB_SUBOPER_ERROR;
break;
@@ -3306,7 +3329,7 @@ Ex_Lob_Error ExLobGlobals::processPreOpens()
lobPtr->openDataCursor(preOpenObj->cursorName_, Lob_Cursor_Simple, preOpenObj->range_,
preOpenObj->bufMaxSize_, preOpenObj->maxBytes_,
- preOpenObj->waited_, this);
+ preOpenObj->waited_, this,0);
}
return LOB_OPER_OK;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index c8be6a9..b05eb8f 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -429,7 +429,7 @@ class ExLob
bool hasNoOpenCursors() { return lobCursors_.empty(); }
Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId);
Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range,
- Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals);
+ Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals, Int32 *hdfsDetailError = NULL);
Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals);
Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId);
Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBinterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp
index 56f7e90..9c3c213 100644
--- a/core/sql/exp/ExpLOBinterface.cpp
+++ b/core/sql/exp/ExpLOBinterface.cpp
@@ -884,7 +884,8 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob,
Int64 &outLen, char * lobData,
Lng32 oper, // 1: open. 2: fetch. 3: close
- Lng32 openType // 0: not applicable. 1: preOpen. 2: mustOpen.
+ Lng32 openType, // 0: not applicable. 1: preOpen. 2: mustOpen.
+ Int32 *hdfsDetailError
)
{
Ex_Lob_Error err;
@@ -942,7 +943,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob,
waitedOp,
exLobGlob,
0,
- NULL, 0,0,0,0,0,0,0,
+ hdfsDetailError, 0,0,0,0,0,0,0,
openType
);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index ed50169..cb6fda8 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -274,7 +274,8 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
Int64 &outLen, char * lobData,
Lng32 oper, // 1: open. 2: fetch. 3: close
- Lng32 openType // 0: not applicable. 1: preOpen. 2: mustOpen.
+ Lng32 openType, // 0: not applicable. 1: preOpen. 2: mustOpen.
+ Int32 *hdfsDetailError = NULL
);
Lng32 ExpLOBinterfaceStats(void * lobGlob,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index 4228108..a0dedea 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -150,9 +150,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRE
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
Rows Processed: 50000
-Task: PREPARATION Status: Ended ET: 00:00:10.156
+Task: PREPARATION Status: Ended ET: 00:00:09.733
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: COMPLETION Status: Ended ET: 00:00:00.404
+Task: COMPLETION Status: Ended ET: 00:00:00.398
--- 50000 row(s) loaded.
>>--
@@ -181,9 +181,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:11.022
+Task: PREPARATION Status: Ended ET: 00:00:12.160
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:00.279
+Task: COMPLETION Status: Ended ET: 00:00:00.362
--- 20000 row(s) loaded.
>>--
@@ -213,9 +213,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOG
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:08.435
+Task: PREPARATION Status: Ended ET: 00:00:08.208
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.266
+Task: COMPLETION Status: Ended ET: 00:00:00.327
--- 20000 row(s) loaded.
>>--
@@ -235,9 +235,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
Rows Processed: 100000
-Task: PREPARATION Status: Ended ET: 00:00:10.885
+Task: PREPARATION Status: Ended ET: 00:00:10.404
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.258
+Task: COMPLETION Status: Ended ET: 00:00:00.332
--- 100000 row(s) loaded.
>>--
@@ -266,9 +266,9 @@ Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SA
Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT
Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
Rows Processed: 160756
-Task: PREPARATION Status: Ended ET: 00:00:13.092
+Task: PREPARATION Status: Ended ET: 00:00:13.314
Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.843
+Task: COMPLETION Status: Ended ET: 00:00:00.309
--- 160756 row(s) loaded.
>>--
@@ -356,10 +356,10 @@ a
+> select * from null_format_src;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.284
+Task: EXTRACT Status: Ended ET: 00:00:00.334
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_default;
@@ -385,10 +385,10 @@ a
+> select * from null_format_src;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.366
+Task: EXTRACT Status: Ended ET: 00:00:00.325
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_empty;
@@ -417,7 +417,7 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.293
+Task: EXTRACT Status: Ended ET: 00:00:00.409
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_colon;
@@ -471,12 +471,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.011
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:03.450
+Task: EXTRACT Status: Ended ET: 00:00:04.200
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.042
+Task: MERGE FILES Status: Ended ET: 00:00:00.064
--- 50000 row(s) unloaded.
>>log;
@@ -509,12 +509,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.976
+Task: EXTRACT Status: Ended ET: 00:00:01.146
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.034
+Task: MERGE FILES Status: Ended ET: 00:00:00.039
--- 20000 row(s) unloaded.
>>log;
@@ -533,12 +533,12 @@ cat /tmp/merged_customer_demogs | wc -l
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.204
+Task: EXTRACT Status: Ended ET: 00:00:00.983
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.031
+Task: MERGE FILES Status: Ended ET: 00:00:00.024
--- 20000 row(s) unloaded.
>>log;
@@ -570,12 +570,12 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.108
+Task: EXTRACT Status: Ended ET: 00:00:01.386
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.040
+Task: MERGE FILES Status: Ended ET: 00:00:00.052
--- 20000 row(s) unloaded.
>>
@@ -596,9 +596,9 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.873
+Task: EXTRACT Status: Ended ET: 00:00:00.911
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.039
+Task: MERGE FILES Status: Ended ET: 00:00:00.049
--- 20000 row(s) unloaded.
>>
@@ -631,10 +631,10 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.803
+Task: EXTRACT Status: Ended ET: 00:00:01.244
--- 20000 row(s) unloaded.
>>
@@ -654,12 +654,12 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file |
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.011
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.838
+Task: EXTRACT Status: Ended ET: 00:00:01.058
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.046
+Task: MERGE FILES Status: Ended ET: 00:00:00.057
--- 20000 row(s) unloaded.
>>
@@ -792,12 +792,12 @@ CD_DEMO_SK CD_GENDER
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.837
+Task: EXTRACT Status: Ended ET: 00:00:00.982
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.042
+Task: MERGE FILES Status: Ended ET: 00:00:00.048
--- 20000 row(s) unloaded.
>>log;
@@ -832,15 +832,15 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/merged* | grep merge
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.836
+Task: EXTRACT Status: Ended ET: 00:00:01.064
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.040
+Task: MERGE FILES Status: Ended ET: 00:00:00.059
--- 20000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_demographics;
(EXPR)
@@ -887,13 +887,13 @@ CD_DEMO_SK CD_GENDER
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.946
+Task: EXTRACT Status: Ended ET: 00:00:00.995
--- 20000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_demographics;
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
@@ -941,13 +941,13 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.325
+Task: EXTRACT Status: Ended ET: 00:00:02.475
--- 50000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_address;
(EXPR)
@@ -997,10 +997,10 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.438
+Task: EXTRACT Status: Ended ET: 00:00:02.595
--- 50000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_address;
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
@@ -1063,10 +1063,10 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:07.083
+Task: EXTRACT Status: Ended ET: 00:00:07.642
--- 100000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer;
(EXPR)
@@ -1114,13 +1114,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>select * from trafodion.hbase.customer_demographics_salt;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.019
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.226
+Task: EXTRACT Status: Ended ET: 00:00:01.228
--- 20000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_demographics;
(EXPR)
@@ -1167,12 +1167,12 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_address where ca_address_sk < 100;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.003
Task: EXTRACT Status: Started
Rows Processed: 99
-Task: EXTRACT Status: Ended ET: 00:00:00.244
+Task: EXTRACT Status: Ended ET: 00:00:00.216
Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.024
+Task: MERGE FILES Status: Ended ET: 00:00:00.023
--- 99 row(s) unloaded.
>>
@@ -1211,10 +1211,10 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 12349
-Task: EXTRACT Status: Ended ET: 00:00:07.776
+Task: EXTRACT Status: Ended ET: 00:00:10.062
--- 12349 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select [first 100] * from hive.hive.unload_store_sales_summary order by ss_sold_date_sk,ss_store_sk;
SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY
@@ -1333,10 +1333,10 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:08.464
+Task: EXTRACT Status: Ended ET: 00:00:09.838
--- 100000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_and_address;
(EXPR)
@@ -1381,13 +1381,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: EXTRACT Status: Started
Rows Processed: 1998
Task: EXTRACT Status: Ended ET: 00:00:00.695
--- 1998 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_address;
(EXPR)
@@ -1495,7 +1495,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111
- snapshot_temp_location /bulkload/20160706204232/
+ snapshot_temp_location /bulkload/20160708212424/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1575,7 +1575,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20160706204243/
+ snapshot_temp_location /bulkload/20160708212436/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1657,11 +1657,11 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111
- snapshot_temp_location /bulkload/20160706204314/
+ snapshot_temp_location /bulkload/20160708212512/
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20160706204314/
+ snapshot_temp_location /bulkload/20160708212512/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1774,13 +1774,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.025
Task: VERIFY SNAPSHO Status: Started
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.425
+Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.494
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.515
+Task: EXTRACT Status: Ended ET: 00:00:03.271
--- 50000 row(s) unloaded.
>>
@@ -1852,13 +1852,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.003
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.002
Task: VERIFY SNAPSHO Status: Started
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.333
+Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.374
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:02.008
+Task: EXTRACT Status: Ended ET: 00:00:02.347
--- 20000 row(s) unloaded.
>>
@@ -1906,16 +1906,16 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.016
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.707
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.520
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.166
+Task: EXTRACT Status: Ended ET: 00:00:01.725
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.026
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.028
--- 20000 row(s) unloaded.
>>
@@ -1970,13 +1970,13 @@ Task: EMPTY TARGET Status: Started
Task: EMPTY TARGET Status: Ended ET: 00:00:00.009
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.515
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.458
Task: EXTRACT Status: Started
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.263
+Task: EXTRACT Status: Ended ET: 00:00:01.347
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.019
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
--- 20000 row(s) unloaded.
>>
@@ -2029,16 +2029,16 @@ CD_DEMO_SK CD_GENDER
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.233
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.079
Task: EXTRACT Status: Started
Rows Processed: 1998
-Task: EXTRACT Status: Ended ET: 00:00:00.847
+Task: EXTRACT Status: Ended ET: 00:00:00.725
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005
--- 1998 row(s) unloaded.
>>
@@ -2113,19 +2113,19 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.048
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
Task: CREATE SNAPSHO Status: Started
Snapshots created: 2
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.709
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:03.077
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:09.569
+Task: EXTRACT Status: Ended ET: 00:00:09.442
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 2
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.010
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.009
--- 100000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_and_address;
(EXPR)
@@ -2190,19 +2190,19 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select c_first_name,c_last_name from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
+Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
Task: CREATE SNAPSHO Status: Started
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.410
+Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.640
Task: EXTRACT Status: Started
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:01.589
+Task: EXTRACT Status: Ended ET: 00:00:01.597
Task: DELETE SNAPSHO Status: Started
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
+Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.007
--- 100000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_name;
(EXPR)
@@ -2272,7 +2272,7 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS;
Task: UNLOAD Status: Started
Task: EXTRACT Status: Started
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.253
+Task: EXTRACT Status: Ended ET: 00:00:02.827
--- 50000 row(s) unloaded.
>>--unload 24 -- should give an error
@@ -2337,10 +2337,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/*
Task: UNLOAD Status: Started
Task: EXTRACT Status: Started
Rows Processed but NOT Written to Disk: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.840
+Task: EXTRACT Status: Ended ET: 00:00:00.904
--- 20000 row(s) unloaded.
->>sh sleep 10;
+>>--sh sleep 10;
>>select count(*) from hive.hive.unload_customer_demographics;
(EXPR)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/249dfefa/core/sql/regress/hive/TEST018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST018 b/core/sql/regress/hive/TEST018
index 0932da5..5f1b611 100644
--- a/core/sql/regress/hive/TEST018
+++ b/core/sql/regress/hive/TEST018
@@ -446,7 +446,7 @@ MERGE FILE 'merged_customer_demographics' OVERWRITE
INTO '/user/hive/exttables/unload_customer_demographics'
select * from trafodion.hbase.customer_demographics_salt
<<+ cardinality 10e10 >>;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_demographics;
select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk < 100 order by cd_demo_sk;
@@ -458,7 +458,7 @@ WITH PURGEDATA FROM TARGET
INTO '/user/hive/exttables/unload_customer_demographics'
select * from trafodion.hbase.customer_demographics_salt
<<+ cardinality 10e10 >>;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_demographics;
select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk <100 order by cd_demo_sk;
@@ -467,7 +467,7 @@ UNLOAD
WITH PURGEDATA FROM TARGET
INTO '/user/hive/exttables/unload_customer_address'
select * from trafodion.hbase.customer_address ;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_address;
select [first 20] * from hive.hive.unload_customer_address where ca_address_sk <100 order by ca_address_sk;
@@ -477,7 +477,7 @@ UNLOAD
WITH PURGEDATA FROM TARGET DELIMITER 124 RECORD_SEPARATOR 10
INTO '/user/hive/exttables/unload_customer_address'
select * from trafodion.hbase.customer_address ;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_address;
select [first 20] * from hive.hive.unload_customer_address where ca_address_sk < 100 order by ca_address_sk;
@@ -492,7 +492,7 @@ WITH PURGEDATA FROM TARGET
--COMPRESSION GZIP
INTO '/user/hive/exttables/unload_customer'
select * from trafodion.hbase.customer_salt;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer;
select [first 20] * from hive.hive.unload_customer where c_customer_sk < 100 order by c_customer_sk;
@@ -503,7 +503,7 @@ WITH PURGEDATA FROM TARGET
--COMPRESSION GZIP
INTO '/user/hive/exttables/unload_customer_demographics'
select * from trafodion.hbase.customer_demographics_salt;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_demographics;
select [first 20] * from hive.hive.unload_customer_demographics where cd_demo_sk < 100 order by cd_demo_sk;
@@ -540,7 +540,7 @@ WITH
PURGEDATA FROM TARGET DELIMITER '|' RECORD_SEPARATOR '\n'
INTO '/user/hive/exttables/unload_store_sales_summary'
select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk;
-sh sleep 10;
+--sh sleep 10;
select [first 100] * from hive.hive.unload_store_sales_summary order by ss_sold_date_sk,ss_store_sk;
--unload 18
@@ -548,7 +548,7 @@ UNLOAD
WITH PURGEDATA FROM TARGET
INTO '/user/hive/exttables/unload_customer_and_address'
select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_and_address;
select [first 20] * from hive.hive.unload_customer_and_address order by ca_address_sk,c_customer_sk;
@@ -558,7 +558,7 @@ WITH
PURGEDATA FROM TARGET
INTO '/user/hive/exttables/unload_customer_address'
select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_address;
select [first 20] * from hive.hive.unload_customer_address order by ca_address_sk;
select [first 20] * from hive.hive.unload_customer_address order by ca_address_sk desc;
@@ -695,7 +695,7 @@ WITH PURGEDATA FROM TARGET
NEW SNAPSHOT HAVING SUFFIX 'SNAP'
INTO '/user/hive/exttables/unload_customer_and_address'
select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_and_address;
select [first 20] * from hive.hive.unload_customer_and_address order by ca_address_sk,c_customer_sk;
@@ -711,7 +711,7 @@ WITH PURGEDATA FROM TARGET
NEW SNAPSHOT HAVING SUFFIX 'SNAP111'
INTO '/user/hive/exttables/unload_customer_name'
select c_first_name,c_last_name from trafodion.hbase.customer_salt;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_name;
select [first 20] * from hive.hive.unload_customer_name order by c_first_name,c_last_name;
@@ -753,7 +753,7 @@ UNLOAD
WITH PURGEDATA FROM TARGET
INTO '/user/hive/exttables/unload_customer_demographics'
(select * from trafodion.hbase.customer_demographics_salt) ;
-sh sleep 10;
+--sh sleep 10;
select count(*) from hive.hive.unload_customer_demographics;
CQD TRAF_UNLOAD_SKIP_WRITING_TO_FILES reset;