You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2017/09/04 06:58:34 UTC
[1/5] incubator-trafodion git commit: Changes to take care of closing
internal cursor statement when extract operation is done and the exact length
of data is passed in to be extracted.
Repository: incubator-trafodion
Updated Branches:
refs/heads/release2.2 2048d253c -> 615b8dd26
Changes to take care of closing internal cursor statement when extract operation is done and the exact length of data is passed in to be extracted.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/89576f3d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/89576f3d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/89576f3d
Branch: refs/heads/release2.2
Commit: 89576f3dc7f5e37977ec3cdd9f0fd5ecadee0d36
Parents: d538162
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Tue Aug 22 17:38:42 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Tue Aug 22 17:38:42 2017 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 2 +-
core/sql/executor/ExExeUtilLoad.cpp | 26 ++------------------------
core/sql/exp/ExpLOBaccess.cpp | 11 +++++++++--
3 files changed, 12 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/89576f3d/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 51695c6..1ca9c97 100644
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -2978,7 +2978,7 @@ public:
char * lobData2_;
Int64 lobDataSpecifiedExtractLen_;
Int64 lobDataLen_;
- Lng32 remainingBytes_;
+ Int64 remainingBytes_;
Lng32 currPos_;
Lng32 numChildRows_;
Int64 requestTag_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/89576f3d/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index dfbbac4..45a15e5 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -3246,7 +3246,7 @@ short ExExeUtilLobExtractTcb::work()
break;
}
- remainingBytes_ = (Lng32)lobDataOutputLen;
+ remainingBytes_ = lobDataOutputLen;
currPos_ = 0;
@@ -3309,33 +3309,11 @@ short ExExeUtilLobExtractTcb::work()
getLobErrStr(intParam1));
step_ = HANDLE_ERROR_;
break;
- }
+ }
step_ = DONE_;
}
break;
-
- case RETURN_STRING_:
- {
- if (qparent_.up->isFull())
- return WORK_OK;
-
- Lng32 size = MINOF((Lng32)lobTdb().dataExtractSizeIOAddr(), (Lng32)remainingBytes_);
-
- moveRowToUpQueue(&lobData_[currPos_], size);
-
- remainingBytes_ -= size;
- currPos_ += size;
-
- if (remainingBytes_ <= 0)
- {
- step_ = READ_CURSOR_;
- qparent_.down->removeHead();
- }
-
- return WORK_RESCHEDULE_AND_RETURN;
- }
- break;
case RETURN_STATUS_:
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/89576f3d/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 3764752..9478c64 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -1949,7 +1949,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
char logBuf[4096];
lobDebugInfo("In ExLob::readCursorData",0,__LINE__,lobTrace_);
- while ( (operLen < tgtSize) && !cursor.eod_ )
+ while ( (operLen <= tgtSize) && !cursor.eod_ )
{
if (cursor.bytesRead_ == cursor.descSize_) // time to read next chunck
@@ -1970,7 +1970,14 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
continue;
}
}
-
+ if (operLen == tgtSize)
+ {
+ //The cursor still has data available but wait until redrive since
+ //we have fetched all the data for this extract operation
+ hdfsCloseFile(fs_, fdData_);
+ fdData_ = NULL;
+ return LOB_OPER_OK;
+ }
bytesAvailable = cursor.descSize_ - cursor.bytesRead_;
bytesToCopy = min(bytesAvailable, tgtSize - operLen);
offset = cursor.descOffset_ + cursor.bytesRead_;
[3/5] incubator-trafodion git commit: Main change:
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/regress/executor/EXPECTED130
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/EXPECTED130 b/core/sql/regress/executor/EXPECTED130
index 87a1d05..6b52203 100644
--- a/core/sql/regress/executor/EXPECTED130
+++ b/core/sql/regress/executor/EXPECTED130
@@ -63,9 +63,9 @@ C1
C1 C2
----------- ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
----------------------------------------
- 1 LOBH000000020001004521773054977751551845217730572979065218212360625565346135020"TRAFODION"."LOB130"
- 2 LOBH000000020001004521773054977751551845217730575426024718212360625592037380020"TRAFODION"."LOB130"
- 3 LOBH000000020001004521773054977751551845217730577394592318212360625613791173020"TRAFODION"."LOB130"
+ 1 LOBH0000000200010789269709250085236319789269709263410318218212370878104855192020"TRAFODION"."LOB130"
+ 2 LOBH0000000200010789269709250085236319789269709264004324418212370878112140465020"TRAFODION"."LOB130"
+ 3 LOBH0000000200010789269709250085236319789269709264603729518212370878118079347020"TRAFODION"."LOB130"
--- 3 row(s) selected.
>>
@@ -654,7 +654,7 @@ And the dish ran away with the fork !
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_txt1.txt');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH000000020001004521773054979487561845217730741742859818212360627257261284020"TRAFODION"."LOB130"
' , 'tlob130_txt1.txt');
+>>extract lobtofile(LOB 'LOBH0000000200010789269709250093030419789269709339542449418212370878867608341020"TRAFODION"."LOB130"
' , 'tlob130_txt1.txt');
Success. Targetfile :tlob130_txt1.txt Length : 19
--- SQL operation complete.
@@ -670,7 +670,7 @@ Success. Targetfile :tlob130_txt1.txt Length : 19
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH000000020001004521773054979514421845217730753331456518212360627372759041020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010789269709250093195519789269709346199639118212370878934176767020"TRAFODION"."LOB130"
' , 'tlob130_deep.jpg');
Success. Targetfile :tlob130_deep.jpg Length : 159018
--- SQL operation complete.
@@ -686,7 +686,7 @@ Success. Targetfile :tlob130_deep.jpg Length : 159018
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush.jpg');/g" >> t130_extract_command;
>>
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH000000020001004521773054979514421845217730753331456518212360627372759041020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200010789269709250093195519789269709346199639118212370878934176767020"TRAFODION"."LOB130"
' , 'tlob130_anoush.jpg');
Success. Targetfile :tlob130_anoush.jpg Length : 230150
--- SQL operation complete.
@@ -847,7 +847,7 @@ Column Name : c2
Input a filename to extract to :
Output File Name : lobc2out.jpg
Extracting lob handle for column c2...
-LOB handle for c2: LOBH000000020001004521773054979514421845217730753331456518212360627372759041020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010789269709250093195519789269709346199639118212370878934176767020"TRAFODION"."LOB130"
Extracting LOB data length for the above handle...
LOB data length :230150
Extracting lob data into file in chunks ...
@@ -921,7 +921,7 @@ And the dish ran away with the spoon.
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_deep2.jpg');/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH000000020002004521773054980105951845217730796607893818212360627806139125020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
+>>extract lobtofile(LOB 'LOBH0000000200020789269709250096808819789269709373537864718212370879207561632020"TRAFODION"."LOB130"
' , 'tlob130_deep2.jpg');
Success. Targetfile :tlob130_deep2.jpg Length : 159018
--- SQL operation complete.
@@ -953,7 +953,7 @@ Hey diddle diddle,
>>sh rm t130_extract_command;
>>sh grep "^LOBH" TMP130 | sed "s/^/extract lobtofile(LOB '/g" | sed "s/$/' , 'tlob130_anoush3.jpg',create,truncate);/g" >> t130_extract_command;
>>obey t130_extract_command;
->>extract lobtofile(LOB 'LOBH000000020003004521773054980180651845217730805462671518212360627894957464020"TRAFODION"."LOB130"
' , 'tlob130_anoush3.jpg',create,truncate);
+>>extract lobtofile(LOB 'LOBH0000000200030789269709250097259519789269709378642018118212370879258524867020"TRAFODION"."LOB130"
' , 'tlob130_anoush3.jpg',create,truncate);
Success. Targetfile :tlob130_anoush3.jpg Length : 230150
--- SQL operation complete.
@@ -1054,12 +1054,12 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT2
ColumnName : C2
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498076106_0001
+ LOB Data File: LOBP_07892697092501002172_0001
LOB EOD : 0
LOB Used Len : 0
ColumnName : C3
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498076106_0002
+ LOB Data File: LOBP_07892697092501002172_0002
LOB EOD : 0
LOB Used Len : 0
ColumnName : C4
@@ -1074,8 +1074,8 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT2
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT2 C2
/user/trafodion/lobs LOBP_00452177305498076106_0001 0 0
-TRAFODION LOB130 TLOB130GT2 C3
/user/trafodion/lobs LOBP_00452177305498076106_0002 0 0
+TRAFODION LOB130 TLOB130GT2 C2
/user/trafodion/lobs LOBP_07892697092501002172_0001 0 0
+TRAFODION LOB130 TLOB130GT2 C3
/user/trafodion/lobs LOBP_07892697092501002172_0002 0 0
TRAFODION LOB130 TLOB130GT2 C4
External HDFS Location External HDFS File 0 0
--- 3 row(s) selected.
@@ -1096,17 +1096,17 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
ColumnName : C2
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0001
+ LOB Data File: LOBP_07892697092500999026_0001
LOB EOD : 15
LOB Used Len : 15
ColumnName : C3
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0002
+ LOB Data File: LOBP_07892697092500999026_0002
LOB EOD : 15
LOB Used Len : 15
ColumnName : C4
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0003
+ LOB Data File: LOBP_07892697092500999026_0003
LOB EOD : 45
LOB Used Len : 45
@@ -1116,9 +1116,9 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT C2
/user/trafodion/lobs LOBP_00452177305498070471_0001 15 15
-TRAFODION LOB130 TLOB130GT C3
/user/trafodion/lobs LOBP_00452177305498070471_0002 15 15
-TRAFODION LOB130 TLOB130GT C4
/user/trafodion/lobs LOBP_00452177305498070471_0003 45 45
+TRAFODION LOB130 TLOB130GT C2
/user/trafodion/lobs LOBP_07892697092500999026_0001 15 15
+TRAFODION LOB130 TLOB130GT C3
/user/trafodion/lobs LOBP_07892697092500999026_0002 15 15
+TRAFODION LOB130 TLOB130GT C4
/user/trafodion/lobs LOBP_07892697092500999026_0003 45 45
--- 3 row(s) selected.
>>delete from tlob130gt where c1=2;
@@ -1135,17 +1135,17 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
ColumnName : C2
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0001
+ LOB Data File: LOBP_07892697092500999026_0001
LOB EOD : 30
LOB Used Len : 25
ColumnName : C3
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0002
+ LOB Data File: LOBP_07892697092500999026_0002
LOB EOD : 31
LOB Used Len : 26
ColumnName : C4
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498070471_0003
+ LOB Data File: LOBP_07892697092500999026_0003
LOB EOD : 71
LOB Used Len : 56
@@ -1155,9 +1155,9 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130GT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130GT C2
/user/trafodion/lobs LOBP_00452177305498070471_0001 30 25
-TRAFODION LOB130 TLOB130GT C3
/user/trafodion/lobs LOBP_00452177305498070471_0002 31 26
-TRAFODION LOB130 TLOB130GT C4
/user/trafodion/lobs LOBP_00452177305498070471_0003 71 56
+TRAFODION LOB130 TLOB130GT C2
/user/trafodion/lobs LOBP_07892697092500999026_0001 30 25
+TRAFODION LOB130 TLOB130GT C3
/user/trafodion/lobs LOBP_07892697092500999026_0002 31 26
+TRAFODION LOB130 TLOB130GT C4
/user/trafodion/lobs LOBP_07892697092500999026_0003 71 56
--- 3 row(s) selected.
>>
@@ -1328,12 +1328,12 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130EXT
ColumnName : C2
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498099204_0001
+ LOB Data File: LOBP_07892697092501013739_0001
LOB EOD : 70
LOB Used Len : 42
ColumnName : C3
Lob Location : /user/trafodion/lobs
- LOB Data File: LOBP_00452177305498099204_0002
+ LOB Data File: LOBP_07892697092501013739_0002
LOB EOD : 125
LOB Used Len : 68
ColumnName : C4
@@ -1348,8 +1348,8 @@ Lob Information for table: "TRAFODION".LOB130.TLOB130EXT
CATALOG_NAME SCHEMA_NAME OBJECT_NAME COLUMN_NAME
LOB_LOCATION LOB_DATA_FILE LOB_DATA_FILE_SIZE_EOD LOB_DATA_FILE_SIZE_USED
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- ---------------------- -----------------------
-TRAFODION LOB130 TLOB130EXT C2
/user/trafodion/lobs LOBP_00452177305498099204_0001 70 42
-TRAFODION LOB130 TLOB130EXT C3
/user/trafodion/lobs LOBP_00452177305498099204_0002 125 68
+TRAFODION LOB130 TLOB130EXT C2
/user/trafodion/lobs LOBP_07892697092501013739_0001 70 42
+TRAFODION LOB130 TLOB130EXT C3
/user/trafodion/lobs LOBP_07892697092501013739_0002 125 68
TRAFODION LOB130 TLOB130EXT C4
External HDFS Location External HDFS File 0 0
--- 3 row(s) selected.
@@ -1366,9 +1366,9 @@ Column Name : c4
Input a filename to extract to :
Output File Name : lobc4ext.txt
Extracting lob handle for column c4...
-LOB handle for c4: LOBH000000080003004521773054980992041845217730910800048818212360628948202356020"TRAFODION"."LOB130"
+LOB handle for c4: LOBH0000000800030789269709250101373919789269709433659319618212370879808793595020"TRAFODION"."LOB130"
Extracting LOB data length for the above handle...
-LOB data length :44
+LOB data length :19
Extracting lob data into file in chunks ...
Wrote 19 bytes to file : lobc4ext.txt
>>sh cat lobc4ext.txt |tee -a LOG130;
@@ -1432,7 +1432,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH000000020001004521773054981739381845217730958615332018212360629426346494020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010789269709250106581019789269709470354926518212370880175708319020"TRAFODION"."LOB130"
>>select lobtostring(c2,20) from t130lob5;
(EXPR)
@@ -1449,7 +1449,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH000000020001004521773054981739381845217730958615332018212360629426346494020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010789269709250106581019789269709470354926518212370880175708319020"TRAFODION"."LOB130"
>>select lobtostring(c2,40) from t130lob5;
(EXPR)
@@ -1466,7 +1466,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH000000020001004521773054981739381845217730958615332018212360629426346494020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010789269709250106581019789269709470354926518212370880175708319020"TRAFODION"."LOB130"
>>select lobtostring(c2,20) from t130lob5;
(EXPR)
@@ -1483,7 +1483,7 @@ Table name : TRAFODION.LOB130.t130lob5
Input lob column name to get handle from :
Column Name : c2
Extracting lob handle for column c2...
-LOB handle for c2: LOBH000000020001004521773054981739381845217730958615332018212360629426346494020"TRAFODION"."LOB130"
+LOB handle for c2: LOBH0000000200010789269709250106581019789269709470354926518212370880175708319020"TRAFODION"."LOB130"
>>select lobtostring(c2,40) from t130lob5;
(EXPR)
@@ -1510,7 +1510,7 @@ zzzzzzzzzzzzzzzzzzzz
C2
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
---------------------------
-LOBH000000020001004521773054980992041845217730978467538918212360629625048479020"TRAFODION"."LOB130"
+LOBH0000000200010789269709250101373919789269709484761570418212370880319729808020"TRAFODION"."LOB130"
--- 1 row(s) selected.
>>-- following should return error since only external lobs will be allowed
@@ -1553,9 +1553,9 @@ LOBH000000020001004521773054980992041845217730978467538918212360629625048479020"
Tables in Schema TRAFODION.LOBSCH
=================================
-LOBDescChunks__00452177305498204482_0001
-LOBDescHandle__00452177305498204482_0001
-LOBMD__00452177305498204482
+LOBDescChunks__07892697092501086222_0001
+LOBDescHandle__07892697092501086222_0001
+LOBMD__07892697092501086222
SB_HISTOGRAMS
SB_HISTOGRAM_INTERVALS
SB_PERSISTENT_SAMPLES
[4/5] incubator-trafodion git commit: Main change:
Posted by sa...@apache.org.
Main change:
Added a method for JDBC driver to be able to close an open cursor abruptly by passing in a 0 length to the extract syntax. This will ensure internal opens are not left around if the application does not retrieve all the data with the stream API.
This abrupt close can be called when the result set is closed or the connection is closed by keeping track of all the open Blob objects. (in the driver)
Added a freeResources method and destructor to LobExtract and LobUpdate Tcbs.
Minor changes and fixes :
Refactored the ExLob::closeCursor
Removed the LOB_CLI_SELECT_CLOSE call from ::fetchCursor so it gets closed only when ExLob::closeCursor is called. Makes the code consistent and readable.
Took care of a few memory allocations and deletions.
Refactored the LOB length calculation into on ecommon method so it works correctly for both regular LOB dat aas well as external LOB files. Earlier the external lobs length was being returned incorrectly. Now it reports the length of the external HDFS file .
Fixed one place in generator where the CQD value was not being multiplied by the MB units correctly.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f1864484
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f1864484
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f1864484
Branch: refs/heads/release2.2
Commit: f18644841f8d3c8aaa5c594c1018798d515821b0
Parents: 8a260f4
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Thu Aug 31 20:03:05 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Thu Aug 31 20:03:05 2017 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 5 +-
core/sql/clitest/blobtest.cpp | 12 +-
core/sql/executor/ExExeUtil.h | 5 +-
core/sql/executor/ExExeUtilLoad.cpp | 229 +++++++++++++++++++----------
core/sql/exp/ExpLOBaccess.cpp | 146 +++++++++++++-----
core/sql/exp/ExpLOBaccess.h | 4 +-
core/sql/exp/ExpLOBenums.h | 4 +-
core/sql/exp/ExpLOBinterface.cpp | 57 ++++++-
core/sql/exp/ExpLOBinterface.h | 13 ++
core/sql/generator/GenRelExeUtil.cpp | 2 +-
core/sql/regress/executor/EXPECTED130 | 78 +++++-----
11 files changed, 394 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/cli/Cli.cpp
----------------------------------------------------------------------
diff --git a/core/sql/cli/Cli.cpp b/core/sql/cli/Cli.cpp
index 1a6eaf5..a25a31b 100644
--- a/core/sql/cli/Cli.cpp
+++ b/core/sql/cli/Cli.cpp
@@ -10261,7 +10261,10 @@ Lng32 SQLCLI_LOBcliInterface
NADELETEBASIC(query, currContext.exHeap());
if (NOT (inCliInterface && (*inCliInterface)))
- delete cliInterface;
+ {
+ delete cliInterface;
+ cliInterface = NULL;
+ }
if (cliRC < 0)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/clitest/blobtest.cpp
----------------------------------------------------------------------
diff --git a/core/sql/clitest/blobtest.cpp b/core/sql/clitest/blobtest.cpp
index a9a457a..492bbd6 100644
--- a/core/sql/clitest/blobtest.cpp
+++ b/core/sql/clitest/blobtest.cpp
@@ -92,7 +92,7 @@ Int32 extractLobToBuffer(CliGlobals *cliglob, char * lobHandle, Int64 &lengthOfL
while ((retcode != 100) && !(retcode<0))
{
retcode = cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen);
- if (!retcode)
+ if (retcode>= 0)
{
memcpy((char*)&(lobFinalBuf[i]),(char *)lobDataBuf,lobExtractLen);
i += lobExtractLen;
@@ -107,6 +107,9 @@ Int32 extractLobToBuffer(CliGlobals *cliglob, char * lobHandle, Int64 &lengthOfL
fclose(lobFileId);
}
+ str_sprintf(query,"extract lobtobuffer(lob '%s', LOCATION %Ld, SIZE 0) ", lobHandle, (Int64)lobDataBuf);
+
+ cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen);
delete lobFinalBuf;
delete query;
delete lobDataBuf;
@@ -142,13 +145,16 @@ Int32 extractLobToFileInChunks(CliGlobals *cliglob, char * lobHandle, char *fil
while ((retcode != 100) && !(retcode<0))
{
retcode = cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen);
- if (!retcode)
+ if (retcode>= 0)
{
byteCount=fwrite(lobDataBuf,sizeof(char),*inputOutputAddr, lobFileId);
cout << "Wrote " << byteCount << " bytes to file : " << filename << endl;
}
}
-
+ lobExtractLen = 0;
+ str_sprintf(query,"extract lobtobuffer(lob '%s', LOCATION %Ld, SIZE %Ld) ", lobHandle, (Int64)lobDataBuf, inputOutputAddr);
+ retcode = cliInterface.executeImmediatePrepare(query);
+ cliInterface.clearExecFetchClose(NULL,NULL,statusBuf, &statusBufLen);
fclose(lobFileId);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 1ca9c97..07d3b22 100644
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -2931,6 +2931,8 @@ public:
ex_globals * glob = 0);
virtual short work();
+ virtual ~ExExeUtilLobExtractTcb();
+ virtual void freeResources();
ExExeUtilLobExtractTdb & lobTdb() const
{
@@ -3036,7 +3038,8 @@ public:
ex_globals * glob = 0);
virtual short work();
-
+ virtual ~ExExeUtilLobUpdateTcb();
+ virtual void freeResources();
ExExeUtilLobUpdateTdb & lobTdb() const
{
return (ExExeUtilLobUpdateTdb &) tdb;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index 45a15e5..614adde 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -2749,6 +2749,45 @@ ExExeUtilLobExtractTcb::ExExeUtilLobExtractTcb
}
+void ExExeUtilLobExtractTcb::freeResources()
+{
+ Lng32 cliRC = 0;
+ Lng32 retcode = 0;
+ void * lobGlobs = getLobGlobals()->lobAccessGlobals();
+ ContextCli *currContext =
+ getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->
+ getStatement()->getContext();
+ //close any open cursors.
+ retcode = ExpLOBInterfaceSelectCursor
+ (lobGlobs,
+ lobName_,
+ lobLoc_,
+ lobType_,
+ lobTdb().getLobHdfsServer(),
+ lobTdb().getLobHdfsPort(),
+
+ lobHandleLen_, lobHandle_,
+ 0, //cursor bytes
+ NULL, //cursor id
+ requestTag_,
+ Lob_Buffer,
+ 0, // not check status
+ 1, // waited op
+
+ 0, lobDataSpecifiedExtractLen_,
+ lobDataLen_, lobData_,
+ 3, // close
+ 0); // open type not applicable
+
+ NADELETE(lobGlobals_,LOBglobals,currContext->exHeap());
+ lobGlobals_ = NULL;
+}
+
+ExExeUtilLobExtractTcb::~ExExeUtilLobExtractTcb()
+{
+ freeResources();
+}
+
short ExExeUtilLobExtractTcb::work()
{
Lng32 cliRC = 0;
@@ -2996,7 +3035,49 @@ short ExExeUtilLobExtractTcb::work()
}
}
-
+ Int16 flags;
+ Lng32 lobNum;
+ Int64 uid, inDescSyskey, descPartnKey;
+ short schNameLen;
+ char schName[1024];
+ ExpLOBoper::extractFromLOBhandle(&flags, &lobType_, &lobNum, &uid,
+ &inDescSyskey, &descPartnKey,
+ &schNameLen, (char *)schName,
+ (char *)lobHandle_, (Lng32)lobHandleLen_);
+
+ //Retrieve the lobLocation for this lobNum which will be used
+ //in the other steps_ which open and read lob data file.
+ short *lobNumList = new (getHeap()) short[1];
+ short *lobTypList = new (getHeap()) short[1];
+ char **lobLocList = new (getHeap()) char*[1];
+ char **lobColNameList = new (getHeap()) char*[1];
+ lobLocList[0] = new (getHeap()) char[1024];
+ lobColNameList[0] = new (getHeap()) char[256];
+
+ Lng32 numLobs = lobNum;
+ Lng32 cliRC = SQL_EXEC_LOBddlInterface
+ (
+ schName,
+ schNameLen,
+ uid,
+ numLobs,
+ LOB_CLI_SELECT_UNIQUE,
+ lobNumList,
+ lobTypList,
+ lobLocList,lobColNameList,lobTdb().getLobHdfsServer(),
+ lobTdb().getLobHdfsPort(),0,FALSE);
+ if (cliRC < 0)
+ {
+ getDiagsArea()->mergeAfter(diags);
+
+ step_ = HANDLE_ERROR_;
+ break;
+ }
+
+ strcpy(lobLoc_, lobLocList[0]);
+ NADELETEBASIC(lobColNameList[0],getHeap());
+ NADELETEBASIC(lobNumList,getHeap());
+ NADELETEBASIC(lobTypList,getHeap());
if (lobTdb().getToType() == ComTdbExeUtilLobExtract::TO_BUFFER_)
step_ = EXTRACT_LOB_DATA_;
else
@@ -3029,28 +3110,31 @@ short ExExeUtilLobExtractTcb::work()
&schNameLen, (char *)schName,
(char *)lobHandle_, (Lng32)lobHandleLen_);
-
- //Retrieve the total length of this lob using the handle info and return to the caller
- Int64 dummy = 0;
- cliRC = SQL_EXEC_LOBcliInterface(lobHandle_, lobHandleLen_,NULL,NULL,NULL,NULL,LOB_CLI_SELECT_LOBLENGTH,LOB_CLI_ExecImmed, 0,&lobDataLen_, &dummy, &dummy,0,0,FALSE);
- if (cliRC < 0)
- {
- getDiagsArea()->mergeAfter(diags);
-
- step_ = HANDLE_ERROR_;
- break;
- }
- if (lobTdb().retrieveLength())
- {
- if ((lobTdb().getBufAddr() != -1) && (lobTdb().getBufAddr() != 0))
- str_cpy_all((char *)lobTdb().getBufAddr(), (char *)&lobDataLen_,sizeof(Int64));
- str_sprintf(statusString_," LOB Length : %d", lobDataLen_);
- step_ = RETURN_STATUS_;
- break;
- }
- else
- step_ = EXTRACT_LOB_DATA_;
- break;
+
+ lobName_ = ExpLOBoper::ExpGetLOBname(uid, lobNum, lobNameBuf_, 1000);
+
+ //Retrieve the total length of this lob using the handle info and return to the caller
+
+ retcode = ExpLOBInterfaceGetLobLength( lobGlobs,
+ lobName_,
+ lobLoc_,
+ lobType_,
+ lobTdb().getLobHdfsServer(),
+ lobTdb().getLobHdfsPort(),
+ lobHandleLen_, lobHandle_,
+ lobDataLen_);
+
+ if (lobTdb().retrieveLength())
+ {
+ if ((lobTdb().getBufAddr() != -1) && (lobTdb().getBufAddr() != 0))
+ str_cpy_all((char *)lobTdb().getBufAddr(), (char *)&lobDataLen_,sizeof(Int64));
+ str_sprintf(statusString_," LOB Length : %d", lobDataLen_);
+ step_ = RETURN_STATUS_;
+ break;
+ }
+ else
+ step_ = EXTRACT_LOB_DATA_;
+ break;
}
case EXTRACT_LOB_DATA_ :
@@ -3069,38 +3153,16 @@ short ExExeUtilLobExtractTcb::work()
lobDataSpecifiedExtractLen_ = lobTdb().totalBufSize_;
-
+ if (lobDataSpecifiedExtractLen_ == 0)
+ {
+ // Passed in length is 0 indicates the caller is done with
+ // this lobhandle and wants to close this cursor
+ step_ = CLOSE_CURSOR_;
+ break;
+ }
- short *lobNumList = new (getHeap()) short[1];
- short *lobTypList = new (getHeap()) short[1];
- char **lobLocList = new (getHeap()) char*[1];
- char **lobColNameList = new (getHeap()) char*[1];
- lobLocList[0] = new (getHeap()) char[1024];
- lobColNameList[0] = new (getHeap()) char[256];
-
- Lng32 numLobs = lobNum;
- Lng32 cliRC = SQL_EXEC_LOBddlInterface
- (
- schName,
- schNameLen,
- uid,
- numLobs,
- LOB_CLI_SELECT_UNIQUE,
- lobNumList,
- lobTypList,
- lobLocList,lobColNameList,lobTdb().getLobHdfsServer(),
- lobTdb().getLobHdfsPort(),0,FALSE);
- if (cliRC < 0)
- {
- getDiagsArea()->mergeAfter(diags);
-
- step_ = HANDLE_ERROR_;
- break;
- }
-
- strcpy(lobLoc_, lobLocList[0]);
-
+
// Read the lob contents into target file
@@ -3204,6 +3266,15 @@ short ExExeUtilLobExtractTcb::work()
if (lobTdb().getToType() == ComTdbExeUtilLobExtract::TO_BUFFER_)
so = Lob_Buffer;
lobDataSpecifiedExtractLen_ = *((Int64 *)(lobTdb().dataExtractSizeIOAddr()));
+
+ if (lobDataSpecifiedExtractLen_ == 0)
+ {
+ // Passed in length is 0 indicates the caller is done with
+ // this lobhandle and wants to close this cursor
+ step_ = CLOSE_CURSOR_;
+ break;
+ }
+
retcode = ExpLOBInterfaceSelectCursor
(lobGlobs,
lobName_,
@@ -3420,6 +3491,18 @@ ExExeUtilLobUpdateTcb::ExExeUtilLobUpdateTcb
currContext->exHeap(),currContext,lobTdb().getLobHdfsServer(),
lobTdb().getLobHdfsPort());
}
+ExExeUtilLobUpdateTcb::~ExExeUtilLobUpdateTcb()
+{
+ freeResources();
+}
+
+void ExExeUtilLobUpdateTcb::freeResources()
+{
+ ContextCli *currContext =
+ getGlobals()->castToExExeStmtGlobals()->castToExMasterStmtGlobals()->
+ getStatement()->getContext();
+ NADELETE(lobGlobals_,LOBglobals,currContext->exHeap());
+}
short ExExeUtilLobUpdateTcb::work()
{
@@ -3538,7 +3621,7 @@ short ExExeUtilLobUpdateTcb::work()
char **lobColNameList = new (getHeap()) char*[1];
lobLocList[0] = new (getHeap()) char[1024];
lobColNameList[0] = new (getHeap()) char[256];
-
+
Lng32 numLobs = lobNum;
Lng32 cliRC = SQL_EXEC_LOBddlInterface
(
@@ -3560,6 +3643,10 @@ short ExExeUtilLobUpdateTcb::work()
}
strcpy(lobLoc_, lobLocList[0]);
+ NADELETEBASIC(lobLocList[0],getHeap());
+ NADELETEBASIC(lobColNameList[0],getHeap());
+ NADELETEBASIC(lobNumList,getHeap());
+ NADELETEBASIC(lobTypList,getHeap());
char outLobHandle[LOB_HANDLE_LEN];
Int32 outHandleLen;
Int64 requestTag = 0;
@@ -3652,6 +3739,10 @@ short ExExeUtilLobUpdateTcb::work()
}
strcpy(lobLoc_, lobLocList[0]);
+ NADELETEBASIC(lobLocList[0],getHeap());
+ NADELETEBASIC(lobColNameList[0],getHeap());
+ NADELETEBASIC(lobNumList,getHeap());
+ NADELETEBASIC(lobTypList,getHeap());
char outLobHandle[LOB_HANDLE_LEN];
Int32 outHandleLen;
Int64 requestTag = 0;
@@ -3744,33 +3835,15 @@ short ExExeUtilLobUpdateTcb::work()
}
strcpy(lobLoc_, lobLocList[0]);
+ NADELETEBASIC(lobLocList[0],getHeap());
+ NADELETEBASIC(lobColNameList[0],getHeap());
+ NADELETEBASIC(lobNumList,getHeap());
+ NADELETEBASIC(lobTypList,getHeap());
+
char outLobHandle[LOB_HANDLE_LEN];
Int32 outHandleLen;
Int64 requestTag = 0;
- /* retcode = ExpLOBInterfaceDelete(lobGlobs,
- lobTdb().getLobHdfsServer(),
- lobTdb().getLobHdfsPort(),
- lobName_,
- lobLoc_,
- lobHandleLen_,
- lobHandle_,
- requestTag_,
- getLobGlobals()->xnId(),
- inDescSyskey,
- 0,1);
- if (retcode < 0)
- {
- Lng32 cliError = 0;
-
- Lng32 intParam1 = -retcode;
- ComDiagsArea * diagsArea = getDiagsArea();
- ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(8442), NULL, &intParam1,
- &cliError, NULL, (char*)"ExpLOBInterfaceUpdate",
- getLobErrStr(intParam1));
- step_ = HANDLE_ERROR_;
- break;
- } */
+
retcode = ExpLOBInterfaceUpdate(lobGlobs,
lobTdb().getLobHdfsServer(),
lobTdb().getLobHdfsPort(),
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 9478c64..af2846c 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -230,21 +230,6 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
if (cliErr == 100 )
{
isEOD= TRUE;
- cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleLenIn,
- NULL, NULL,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- &cliInterface,
- transId,lobTrace_);
- if (cliErr <0 )
- {
- str_sprintf(logBuf, "LOB_CLI_SELECT_CLOSE Returned cli error %d",cliErr);
- lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
- err = LOB_DESC_READ_ERROR;
- return err;
- }
}
else
@@ -256,9 +241,17 @@ Ex_Lob_Error ExLob::fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOff
char temp[blackBoxLen+1];
str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE);
lobDataFile_ = temp;
+ outOffset = offset;
+ err=statSourceFile(temp,outSize);
+ if (err != LOB_OPER_OK)
+ return err;
}
- outOffset = offset;
- outSize = size;
+ else
+ {
+ outOffset = offset;
+ outSize = size;
+ }
+
}
str_sprintf(logBuf, " Returned after ::fetchCursor %Ld,%Ld",outOffset,outSize);
@@ -804,6 +797,62 @@ Ex_Lob_Error ExLob::readExternalSourceFile(char *srcfile, char *&fileData, Int32
return LOB_OPER_OK;
}
+Ex_Lob_Error ExLob::getLength(char *handleIn, Int32 handleInLen,Int64 &outLobLen,LobsSubOper so, Int64 transId)
+{
+ char logBuf[4096];
+ Int32 cliErr = 0;
+ Ex_Lob_Error err=LOB_OPER_OK;
+ char *blackBox = new(getLobGlobalHeap()) char[MAX_LOB_FILE_NAME_LEN+6];
+ Int32 blackBoxLen = 0;
+ Int64 dummy = 0;
+ Int32 dummy2 = 0;
+ if (so != Lob_External_File)
+ {
+
+ cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,NULL,NULL,NULL,NULL,LOB_CLI_SELECT_LOBLENGTH,LOB_CLI_ExecImmed, 0,&outLobLen, 0, 0,0,transId,lobTrace_);
+
+ if (cliErr < 0 ) {
+ str_sprintf(logBuf,"CLI SELECT_LOBLENGTH returned error %d",cliErr);
+ lobDebugInfo(logBuf, 0,__LINE__,lobTrace_);
+
+ return LOB_DESC_READ_ERROR;
+ }
+ }
+ else
+ {
+ //Get the lob external filename from the descriptor file and get the length of the file
+ cliErr = SQL_EXEC_LOBcliInterface(handleIn,
+ handleInLen,
+ blackBox, &blackBoxLen,
+ NULL, 0,
+ LOB_CLI_SELECT_UNIQUE, LOB_CLI_ExecImmed,
+ &dummy, &dummy,
+ &dummy, &dummy,
+ 0,
+ transId,lobTrace_);
+ if (cliErr < 0 ) {
+ str_sprintf(logBuf,"CLI SELECT_LOBLENGTH returned error %d",cliErr);
+ lobDebugInfo(logBuf, 0,__LINE__,lobTrace_);
+
+ return LOB_DESC_READ_ERROR;
+ }
+ if (blackBox && blackBoxLen >0 )
+ {
+ // we have received the external data file name from the
+ // descriptor table
+
+ char temp[blackBoxLen+1];
+ str_cpy_and_null(temp, blackBox, blackBoxLen, '\0', '0', TRUE);
+
+
+ err=statSourceFile(temp,outLobLen);
+ if (err != LOB_OPER_OK)
+ return err;
+ }
+
+ }
+ return err;
+}
Ex_Lob_Error ExLob::writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOper, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char *handleOut, Int32 &handleOutLen, Int64 xnId, void *lobGlobals)
{
Ex_Lob_Error err=LOB_OPER_OK;
@@ -1566,7 +1615,7 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 h
if (cursor.eod_) {
// remove cursor from the map.
// server has already closed the cursor.
- closeCursor(handleIn, handleInLen);
+ closeCursor(handleIn, handleInLen,transId);
// indicate EOD to SQL
operLen = 0;
return LOB_OPER_OK;
@@ -1585,17 +1634,48 @@ Ex_Lob_Error ExLob::readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 h
-Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int32 handleInLen)
+Ex_Lob_Error ExLob::closeCursor(char *handleIn, Int32 handleInLen, Int64 transId)
{
char logBuf[4096];
+ Int64 dummyParam = 0;
+ Int32 cliErr = 0;
+ Ex_Lob_Error err = LOB_OPER_OK;
lobCursors_it it = lobCursors_.find(string(handleIn, handleInLen));
+ if (it == lobCursors_.end())
+ {
+ // cursor already closed
+ return LOB_OPER_OK;
+ }
+
+ void *cliInterface = it->second.cliInterface_;
+ if (cliInterface)
+ {
+ cliErr = SQL_EXEC_LOBcliInterface(handleIn, handleInLen,
+ NULL, NULL,
+ (char *)&dummyParam, (Lng32 *)&dummyParam,
+ LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed,
+ &dummyParam, &dummyParam,
+ &dummyParam, &dummyParam,
+ &cliInterface,
+ transId,lobTrace_);
+ if (cliErr <0 )
+ {
+ str_sprintf(logBuf, "LOB_CLI_SELECT_CLOSE Returned cli error %d",cliErr);
+ lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
+ err = LOB_DESC_READ_ERROR;
+ return err;
+ }
+ }
if (it != lobCursors_.end())
{
str_sprintf(logBuf,"closing cursor for handle");
lobDebugInfo(logBuf,0,__LINE__,lobTrace_);
lobCursors_.erase(it);
}
- return LOB_OPER_OK;
+
+
+
+ return LOB_OPER_OK;
}
Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset, Int64 lobMaxSize, Int64 lobMaxChunkMemLen, char *handleIn, Int32 handleInLen, Int64 lobGCLimit, void *lobGlobals)
@@ -1949,7 +2029,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
char logBuf[4096];
lobDebugInfo("In ExLob::readCursorData",0,__LINE__,lobTrace_);
- while ( (operLen <= tgtSize) && !cursor.eod_ )
+ while ( (operLen < tgtSize) && !cursor.eod_ )
{
if (cursor.bytesRead_ == cursor.descSize_) // time to read next chunck
@@ -1970,14 +2050,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
continue;
}
}
- if (operLen == tgtSize)
- {
- //The cursor still has data available but wait until redrive since
- //we have fetched all the data for this extract operation
- hdfsCloseFile(fs_, fdData_);
- fdData_ = NULL;
- return LOB_OPER_OK;
- }
+
bytesAvailable = cursor.descSize_ - cursor.bytesRead_;
bytesToCopy = min(bytesAvailable, tgtSize - operLen);
offset = cursor.descOffset_ + cursor.bytesRead_;
@@ -2028,6 +2101,7 @@ Ex_Lob_Error ExLob::readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, I
operLen += bytesRead;
tgt += bytesRead;
}
+
hdfsCloseFile(fs_, fdData_);
fdData_ = NULL;
return LOB_OPER_OK;
@@ -2117,7 +2191,7 @@ Ex_Lob_Error ExLob::readDataToMem(char *memAddr,
if (err==LOB_OPER_OK)
closeCursor(handleIn,
- handleLenIn);
+ handleLenIn,transId);
else
return err;
}
@@ -2232,7 +2306,7 @@ Ex_Lob_Error ExLob::readDataToLocalFile(char *fileName, Int64 offset, Int64 siz
tgtOffset += chunkSize;
}
closeCursor(handleIn,
- handleInLen);
+ handleInLen,transId);
}
close(fdDestFile);
return LOB_OPER_OK;
@@ -2359,7 +2433,7 @@ Ex_Lob_Error ExLob::readDataToHdfsFile(char *tgtFileName, Int64 offset, Int64 s
}
closeCursor(handleIn,
- handleInLen);
+ handleInLen,transId);
}
hdfsCloseFile(fs_, fdTgtFile);
fdTgtFile=NULL;
@@ -2644,7 +2718,11 @@ Ex_Lob_Error ExLobsOper (
if (blackBox)
(lobGlobals->getHeap())->deallocateMemory((char*) blackBox);
break;
-
+ case Lob_GetLength:
+ {
+ err = lobPtr->getLength(handleIn, handleInLen,retOperLen,subOperation,transId);
+ }
+ break;
case Lob_ReadDesc: // read desc only. Needed for pass thru.
err = lobPtr->getDesc(desc,handleIn,handleInLen,(char *)blackBox, &blackBoxLen,handleOut,handleOutLen,transId);
retOperLen = 0;
@@ -2711,7 +2789,7 @@ Ex_Lob_Error ExLobsOper (
break;
case Lob_CloseCursor:
- err = lobPtr->closeCursor(handleIn, handleInLen);
+ err = lobPtr->closeCursor(handleIn, handleInLen,transId);
break;
case Lob_CloseDataCursorSimple:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index acb63fc..3a5a8ca 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -436,7 +436,7 @@ class ExLob : public NABasicObject
Ex_Lob_Error purgeLob();
Ex_Lob_Error closeFile();
LobInputOutputFileType fileType(char *ioFileName);
- Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen);
+ Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen,Int64 transId);
Ex_Lob_Error closeDataCursorSimple(const char *fileName, ExLobGlobals *lobGlobals);
Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage,
@@ -496,7 +496,7 @@ class ExLob : public NABasicObject
ExLobStats *getStats() { return &stats_; }
NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;}
-
+ Ex_Lob_Error getLength(char *handleIn, Int32 handleInLen,Int64 &outLobLen,LobsSubOper so, Int64 transId);
// ExLobRequest *getRequest() { return &request_; }
//The next 2 functions are not active at this point. They serve as an example
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/exp/ExpLOBenums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h
index 37af1d0..d04b080 100644
--- a/core/sql/exp/ExpLOBenums.h
+++ b/core/sql/exp/ExpLOBenums.h
@@ -259,7 +259,9 @@ typedef enum {
Lob_Cleanup, // destroy everything under globals
Lob_PerformGC,
Lob_RestoreLobDataFile,
- Lob_PurgeBackupLobDataFile
+ Lob_PurgeBackupLobDataFile,
+ Lob_GetLength
+
} LobsOper;
typedef enum {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/exp/ExpLOBinterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp
index 5016a01..9b58344 100644
--- a/core/sql/exp/ExpLOBinterface.cpp
+++ b/core/sql/exp/ExpLOBinterface.cpp
@@ -982,8 +982,9 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob,
waitedOp,
exLobGlob,
0,
- hdfsDetailError, 0,0,0,0,0,0,0,
+ hdfsDetailError,0,0,0,0,0,0,0,
openType
+
);
if (err != LOB_OPER_OK)
@@ -994,6 +995,60 @@ Lng32 ExpLOBInterfaceSelectCursor(void * exLobGlob,
return LOB_ACCESS_SUCCESS;
}
+
+Lng32 ExpLOBInterfaceGetLobLength(void * exLobGlob,
+ char * lobName,
+ char * lobLoc,
+ Lng32 lobType,
+ char * lobHdfsServer,
+ Lng32 lobHdfsPort,
+ Int32 handleLen,
+ char * lobHandle,
+ Int64 &outLobLen
+
+ )
+{
+ Ex_Lob_Error err;
+
+ Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
+ Ex_Lob_Error status;
+ Int64 cliError=0;
+
+ LobsOper lo;
+ LobsSubOper so;
+ LobsStorage ls = (LobsStorage)lobType;
+ if (ls == Lob_External_HDFS_File)
+ so = Lob_External_File;
+ else
+ so = Lob_Buffer;
+ err = ExLobsOper(lobName,
+ lobHandle, handleLen,
+ lobHdfsServer, lobHdfsPort,
+ NULL, dummyParam2,
+ dummyParam, dummyParam,
+ outLobLen,
+ dummyParam, dummyParam,
+ status, cliError,
+ lobLoc, ls, //Lob_HDFS_File,
+ NULL, 0,
+ dummyParam,NULL,
+ Lob_GetLength,
+ so,
+ TRUE,
+ exLobGlob,
+ 0,
+ 0, 0,0,0,0,0,0,0,
+ 0
+ );
+
+ if (err != LOB_OPER_OK)
+ {
+ return -err;
+ }
+
+ return LOB_ACCESS_SUCCESS;
+}
Lng32 ExpLOBinterfaceStats(
void * exLobGlob,
ExLobStats * lobStats,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index a8245c8..88b6fe9 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -331,6 +331,19 @@ Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob,
short replication =0,
int blocksize=0);
+Lng32 ExpLOBInterfaceGetLobLength(void * exLobGlob,
+ char * lobName,
+ char * lobLoc,
+ Lng32 lobType,
+ char * lobHdfsServer,
+ Lng32 lobHdfsPort,
+ Int32 handleLen,
+ char * lobHandle,
+ Int64 &outLobLen
+
+ );
+
+
/*
class HdfsFileInfo
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f1864484/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index d514cb5..df627cb 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -4298,7 +4298,7 @@ if (handleInStringFormat_)
{
exe_util_tdb->setRetrieveLength(TRUE);
}
- exe_util_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE));
+ exe_util_tdb->setTotalBufSize(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)*1024*1024);
generator->setCriDesc(givenDesc, Generator::DOWN);
generator->setCriDesc(returnedDesc, Generator::UP);
[5/5] incubator-trafodion git commit: Merge remote branch
'origin/pr/1216/head' into merge_1216
Posted by sa...@apache.org.
Merge remote branch 'origin/pr/1216/head' into merge_1216
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/615b8dd2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/615b8dd2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/615b8dd2
Branch: refs/heads/release2.2
Commit: 615b8dd26d38e19ba17cf9451453bc38e8879dcc
Parents: 2048d25 f186448
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Mon Sep 4 06:58:05 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Mon Sep 4 06:58:05 2017 +0000
----------------------------------------------------------------------
core/sql/cli/Cli.cpp | 5 +-
core/sql/clitest/blobtest.cpp | 12 +-
core/sql/executor/ExExeUtil.h | 7 +-
core/sql/executor/ExExeUtilLoad.cpp | 255 ++++++++-----
core/sql/exp/ExpLOBaccess.cpp | 135 +++++--
core/sql/exp/ExpLOBaccess.h | 4 +-
core/sql/exp/ExpLOBenums.h | 4 +-
core/sql/exp/ExpLOBinterface.cpp | 57 ++-
core/sql/exp/ExpLOBinterface.h | 13 +
core/sql/generator/GenRelExeUtil.cpp | 2 +-
core/sql/regress/executor/EXPECTED130 | 78 ++--
core/sql/regress/hive/EXPECTED018 | 585 +++++++++++++++--------------
12 files changed, 688 insertions(+), 469 deletions(-)
----------------------------------------------------------------------
[2/5] incubator-trafodion git commit: Expected file change to show
result of unload.
Posted by sa...@apache.org.
Expected file change to show result of unload.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/8a260f42
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/8a260f42
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/8a260f42
Branch: refs/heads/release2.2
Commit: 8a260f425e0527100c8039b8db8a8b4db64b1101
Parents: 89576f3
Author: Sandhya Sundaresan <sa...@apache.org>
Authored: Tue Aug 22 17:41:39 2017 +0000
Committer: Sandhya Sundaresan <sa...@apache.org>
Committed: Tue Aug 22 17:41:39 2017 +0000
----------------------------------------------------------------------
core/sql/regress/hive/EXPECTED018 | 585 +++++++++++++++++----------------
1 file changed, 293 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/8a260f42/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index 01eb789..c9d2a4e 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -146,18 +146,18 @@
>>load with no recovery into customer_address
+>select * from hive.hive.customer_address;
Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: CLEANUP Status: Started Time: 2017-05-01 18:46:29.746
-Task: CLEANUP Status: Ended Time: 2017-05-01 18:46:29.768
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.022
-Task: LOADING DATA Status: Started Time: 2017-05-01 18:46:29.768
+Task: CLEANUP Status: Started Time: 2017-08-15 14:53:40.511
+Task: CLEANUP Status: Ended Time: 2017-08-15 14:53:40.528
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.018
+Task: LOADING DATA Status: Started Time: 2017-08-15 14:53:40.529
Rows Processed: 50000
Error Rows: 0
-Task: LOADING DATA Status: Ended Time: 2017-05-01 18:46:39.501
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:09.236
-Task: COMPLETION Status: Started Time: 2017-05-01 18:46:39.509
+Task: LOADING DATA Status: Ended Time: 2017-08-15 14:53:50.229
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:09.701
+Task: COMPLETION Status: Started Time: 2017-08-15 14:53:50.229
Rows Loaded: 50000
-Task: COMPLETION Status: Ended Time: 2017-05-01 18:46:39.346
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.341
+Task: COMPLETION Status: Ended Time: 2017-08-15 14:53:50.635
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.406
--- 50000 row(s) loaded.
>>--
@@ -182,18 +182,18 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.341
>>load with no recovery into customer_demographics
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 20000;
Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Started Time: 2017-05-01 18:46:42.390
-Task: CLEANUP Status: Ended Time: 2017-05-01 18:46:42.404
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.014
-Task: LOADING DATA Status: Started Time: 2017-05-01 18:46:42.404
+Task: CLEANUP Status: Started Time: 2017-08-15 14:53:54.217
+Task: CLEANUP Status: Ended Time: 2017-08-15 14:53:54.237
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.020
+Task: LOADING DATA Status: Started Time: 2017-08-15 14:53:54.237
Rows Processed: 20000
Error Rows: 0
-Task: LOADING DATA Status: Ended Time: 2017-05-01 18:46:55.199
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:12.795
-Task: COMPLETION Status: Started Time: 2017-05-01 18:46:55.199
+Task: LOADING DATA Status: Ended Time: 2017-08-15 14:54:07.423
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:13.186
+Task: COMPLETION Status: Started Time: 2017-08-15 14:54:07.423
Rows Loaded: 20000
-Task: COMPLETION Status: Ended Time: 2017-05-01 18:46:55.606
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.407
+Task: COMPLETION Status: Ended Time: 2017-08-15 14:54:07.866
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.443
--- 20000 row(s) loaded.
>>--
@@ -219,18 +219,18 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.407
>>load with no recovery into customer_demographics_salt
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 20000;
Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Started Time: 2017-05-01 18:47:00.293
-Task: CLEANUP Status: Ended Time: 2017-05-01 18:47:00.311
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.018
-Task: LOADING DATA Status: Started Time: 2017-05-01 18:47:00.311
+Task: CLEANUP Status: Started Time: 2017-08-15 14:54:12.393
+Task: CLEANUP Status: Ended Time: 2017-08-15 14:54:12.404
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.011
+Task: LOADING DATA Status: Started Time: 2017-08-15 14:54:12.404
Rows Processed: 20000
Error Rows: 0
-Task: LOADING DATA Status: Ended Time: 2017-05-01 18:47:10.603
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:10.292
-Task: COMPLETION Status: Started Time: 2017-05-01 18:47:10.603
+Task: LOADING DATA Status: Ended Time: 2017-08-15 14:54:22.330
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:09.628
+Task: COMPLETION Status: Started Time: 2017-08-15 14:54:22.331
Rows Loaded: 20000
-Task: COMPLETION Status: Ended Time: 2017-05-01 18:47:11.346
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.431
+Task: COMPLETION Status: Ended Time: 2017-08-15 14:54:22.384
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.352
--- 20000 row(s) loaded.
>>--
@@ -246,18 +246,18 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.431
>>load with no recovery into customer_salt
+>select * from hive.hive.customer;
Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: CLEANUP Status: Started Time: 2017-05-01 18:47:13.570
-Task: CLEANUP Status: Ended Time: 2017-05-01 18:47:13.584
+Task: CLEANUP Status: Started Time: 2017-08-15 14:54:25.112
+Task: CLEANUP Status: Ended Time: 2017-08-15 14:54:25.127
Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.015
-Task: LOADING DATA Status: Started Time: 2017-05-01 18:47:13.584
+Task: LOADING DATA Status: Started Time: 2017-08-15 14:54:25.127
Rows Processed: 100000
Error Rows: 0
-Task: LOADING DATA Status: Ended Time: 2017-05-01 18:47:25.306
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:11.722
-Task: COMPLETION Status: Started Time: 2017-05-01 18:47:25.306
+Task: LOADING DATA Status: Ended Time: 2017-08-15 14:54:38.572
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:13.445
+Task: COMPLETION Status: Started Time: 2017-08-15 14:54:38.572
Rows Loaded: 100000
-Task: COMPLETION Status: Ended Time: 2017-05-01 18:47:25.645
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.339
+Task: COMPLETION Status: Ended Time: 2017-08-15 14:54:38.978
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.406
--- 100000 row(s) loaded.
>>--
@@ -282,18 +282,18 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.339
>>load with no recovery into store_sales_salt
+>select * from hive.hive.store_sales where ss_item_sk <= 1000;
Task: LOAD Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: CLEANUP Status: Started Time: 2017-05-01 18:47:28.700
-Task: CLEANUP Status: Ended Time: 2017-05-01 18:47:28.717
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.017
-Task: LOADING DATA Status: Started Time: 2017-05-01 18:47:28.717
+Task: CLEANUP Status: Started Time: 2017-08-15 14:54:43.982
+Task: CLEANUP Status: Ended Time: 2017-08-15 14:54:43.114
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.016
+Task: LOADING DATA Status: Started Time: 2017-08-15 14:54:43.114
Rows Processed: 160756
Error Rows: 0
-Task: LOADING DATA Status: Ended Time: 2017-05-01 18:47:42.429
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:13.711
-Task: COMPLETION Status: Started Time: 2017-05-01 18:47:42.429
+Task: LOADING DATA Status: Ended Time: 2017-08-15 14:54:57.277
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:14.163
+Task: COMPLETION Status: Started Time: 2017-08-15 14:54:57.277
Rows Loaded: 160756
-Task: COMPLETION Status: Ended Time: 2017-05-01 18:47:42.753
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.324
+Task: COMPLETION Status: Ended Time: 2017-08-15 14:54:57.644
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.367
--- 160756 row(s) loaded.
>>--
@@ -380,13 +380,13 @@ a
+> into '/user/trafodion/hive/exttables/null_format_default'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:47:51.294
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:47:51.296
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
-Task: EXTRACT Status: Started Time: 2017-05-01 18:47:51.296
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:55:09.437
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:55:09.493
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2017-08-15 14:55:09.494
Rows Processed: 10
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:47:51.439
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.142
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:55:09.218
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.169
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_default;
@@ -411,13 +411,13 @@ a
+> into '/user/trafodion/hive/exttables/null_format_empty'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:47:52.194
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:47:52.195
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
-Task: EXTRACT Status: Started Time: 2017-05-01 18:47:52.195
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:55:11.384
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:55:11.390
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2017-08-15 14:55:11.390
Rows Processed: 10
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:47:52.297
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.102
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:55:11.610
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.220
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_empty;
@@ -442,13 +442,13 @@ a ?
+> into '/user/trafodion/hive/exttables/null_format_colon'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:47:53.342
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:47:53.357
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.001
-Task: EXTRACT Status: Started Time: 2017-05-01 18:47:53.358
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:55:12.295
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:55:12.304
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
+Task: EXTRACT Status: Started Time: 2017-08-15 14:55:12.304
Rows Processed: 10
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:47:53.305
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.270
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:55:12.440
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.136
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_colon;
@@ -510,16 +510,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_address
+>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:01.614
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:01.615
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.001
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:01.615
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:07.950
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:07.102
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:07.102
Rows Processed: 50000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:02.955
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.340
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:02.955
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:02.995
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.040
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:08.375
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.274
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:08.375
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:08.429
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.053
--- 50000 row(s) unloaded.
>>log;
@@ -551,16 +551,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:06.594
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:06.602
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.008
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:06.602
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:12.216
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:12.226
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:12.226
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:07.141
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.539
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:07.142
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:07.178
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.037
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:12.740
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.513
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:12.740
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:12.792
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.052
--- 20000 row(s) unloaded.
>>log;
@@ -578,16 +578,16 @@ cat /tmp/merged_customer_demogs | wc -l
+>select * from trafodion.hbase.customer_demographics
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:10.514
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:10.521
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:10.521
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:15.875
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:15.887
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.012
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:15.887
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:10.860
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.339
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:10.860
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:10.903
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.043
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:16.341
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.454
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:16.341
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:16.380
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.039
--- 20000 row(s) unloaded.
>>log;
@@ -618,21 +618,22 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:16.748
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:16.761
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.001
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:16.761
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:21.934
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:21.941
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:21.941
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:17.195
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.943
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:17.196
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:17.724
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.053
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:23.323
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.061
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:23.328
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:23.624
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.059
--- 20000 row(s) unloaded.
>>
>>log;
regrhadoop.ksh fs -du -s /user/trafodion/bulkload/customer_demographics_salt/merged_customer_demogs_3
+778224 778224 /user/trafodion/bulkload/customer_demographics_salt/merged_customer_demogs_3
>>-------------------
>>--unload 5
>>UNLOAD
@@ -643,16 +644,16 @@ regrhadoop.ksh fs -du -s /user/trafodion/bulkload/customer_demographics_salt/mer
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:20.244
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:20.247
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.003
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:20.247
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:26.136
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:26.145
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:26.145
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:20.801
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.554
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:20.801
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:20.849
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.048
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:26.720
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.575
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:26.720
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:26.785
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.065
--- 20000 row(s) unloaded.
>>
@@ -684,13 +685,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:24.752
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:24.849
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:30.146
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:30.248
Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:24.850
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:30.248
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:24.711
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.626
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:30.457
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.432
--- 20000 row(s) unloaded.
>>
@@ -709,16 +710,16 @@ regrhadoop.ksh fs -ls /user/trafodion/bulkload/customer_demographics_salt/file*
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:30.834
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:30.854
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.020
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:30.854
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:36.164
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:36.180
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.017
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:36.180
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:31.428
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.574
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:31.428
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:31.517
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.089
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:36.616
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.435
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:36.616
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:36.675
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.060
--- 20000 row(s) unloaded.
>>
@@ -850,16 +851,16 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:42.894
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:42.904
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:42.904
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:47.463
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:47.531
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:47.531
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:43.534
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.630
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:43.534
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:43.583
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.049
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:47.930
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.877
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:47.930
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:47.997
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.067
--- 20000 row(s) unloaded.
>>log;
@@ -893,16 +894,16 @@ regrhadoop.ksh fs -ls /user/trafodion/bulkload/customer_demographics_salt/merged
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:46.855
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:46.863
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.008
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:46.863
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:50.956
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:50.966
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:50.966
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:47.716
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.852
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:49:47.716
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:49:47.799
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.083
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:51.783
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.817
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:56:51.783
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:56:51.867
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.084
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -951,13 +952,13 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:49.955
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:49.962
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:49.962
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:54.325
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:54.331
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:54.331
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:50.550
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.588
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:54.845
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.514
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -965,7 +966,7 @@ Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.588
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1493664587, failedModTS = 1493664590, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1502809011, failedModTS = 1502809014, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
(EXPR)
--------------------
@@ -1007,13 +1008,13 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/trafodion/hive/exttables/unload_customer_address'
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:53.855
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:53.858
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.003
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:53.858
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:56:57.715
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:56:57.722
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:56:57.722
Rows Processed: 50000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:54.791
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.933
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:56:58.660
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.938
--- 50000 row(s) unloaded.
>>--sh sleep 10;
@@ -1062,13 +1063,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/trafodion/hive/exttables/unload_customer_address'
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:49:57.416
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:49:57.421
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
-Task: EXTRACT Status: Started Time: 2017-05-01 18:49:57.422
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:02.463
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:02.470
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:02.470
Rows Processed: 50000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:49:58.307
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.885
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:03.210
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.740
--- 50000 row(s) unloaded.
>>--sh sleep 10;
@@ -1076,7 +1077,7 @@ Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.885
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1493664594, failedModTS = 1493664597, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_address
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1502809018, failedModTS = 1502809022, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_address
(EXPR)
--------------------
@@ -1130,13 +1131,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/trafodion/hive/exttables/unload_customer'
+>select * from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:01.495
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:01.497
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:01.497
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:08.695
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:08.701
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:08.701
Rows Processed: 100000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:04.887
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:03.390
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:12.919
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:04.217
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -1186,13 +1187,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>INTO '/user/trafodion/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:07.724
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:07.734
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:07.734
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:15.925
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:15.936
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.012
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:15.937
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:08.712
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.337
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:16.236
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.299
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -1241,16 +1242,16 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/trafodion/bulkload/customer_address'
+>select * from trafodion.hbase.customer_address where ca_address_sk < 100;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:10.860
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:10.909
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:10.910
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:18.175
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:18.182
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:18.182
Rows Processed: 99
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:10.159
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.068
-Task: MERGE FILES Status: Started Time: 2017-05-01 18:50:10.159
-Task: MERGE FILES Status: Ended Time: 2017-05-01 18:50:10.193
-Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.034
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:18.248
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.065
+Task: MERGE FILES Status: Started Time: 2017-08-15 14:57:18.248
+Task: MERGE FILES Status: Ended Time: 2017-08-15 14:57:18.284
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.036
--- 99 row(s) unloaded.
>>
@@ -1285,13 +1286,13 @@ regrhadoop.ksh fs -rm /user/trafodion/hive/exttables/unload_customer_demographic
+>INTO '/user/trafodion/hive/exttables/unload_store_sales_summary'
+>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:13.905
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:13.912
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.008
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:13.912
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:21.794
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:21.809
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.014
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:21.809
Rows Processed: 12349
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:18.119
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:04.207
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:26.261
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:04.452
--- 12349 row(s) unloaded.
>>--sh sleep 10;
@@ -1409,13 +1410,13 @@ SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY
+>INTO '/user/trafodion/hive/exttables/unload_customer_and_address'
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:20.184
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:20.186
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.003
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:20.186
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:28.396
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:28.406
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:28.406
Rows Processed: 100000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:23.138
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:02.952
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:31.408
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:03.002
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -1462,13 +1463,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>INTO '/user/trafodion/hive/exttables/unload_customer_address'
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:50:27.804
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:50:27.810
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
-Task: EXTRACT Status: Started Time: 2017-05-01 18:50:27.810
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 14:57:36.424
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 14:57:36.432
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.008
+Task: EXTRACT Status: Started Time: 2017-08-15 14:57:36.432
Rows Processed: 1998
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:50:28.224
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.415
+Task: EXTRACT Status: Ended Time: 2017-08-15 14:57:36.875
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.443
--- 1998 row(s) unloaded.
>>--sh sleep 10;
@@ -1579,7 +1580,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111
- snapshot_temp_location /user/trafodion/bulkload/20170501185033/
+ snapshot_temp_location /user/trafodion/bulkload/20170815145742/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1663,7 +1664,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /user/trafodion/bulkload/20170501185041/
+ snapshot_temp_location /user/trafodion/bulkload/20170815145852/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1758,12 +1759,12 @@ ESP_EXCHANGE ============================== SEQ_NO 6 ONLY CHILD 5
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111
- snapshot_temp_location /user/trafodion/bulkload/20170501185103/
+ snapshot_temp_location /user/trafodion/bulkload/20170815145917/
ESP_EXCHANGE ============================== SEQ_NO 2 ONLY CHILD 1
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /user/trafodion/bulkload/20170501185103/
+ snapshot_temp_location /user/trafodion/bulkload/20170815145917/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1881,17 +1882,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from customer_address
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:52:43.214
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:52:43.224
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
-Task: VERIFY SNAPSHO Status: Started Time: 2017-05-01 18:52:43.224
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:14.544
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:14.550
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: VERIFY SNAPSHO Status: Started Time: 2017-08-15 15:02:14.550
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended Time: 2017-05-01 18:52:43.607
-Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.383
-Task: EXTRACT Status: Started Time: 2017-05-01 18:52:43.607
+Task: VERIFY SNAPSHO Status: Ended Time: 2017-08-15 15:02:15.334
+Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.784
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:15.334
Rows Processed: 50000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:52:44.813
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.206
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:16.637
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.302
--- 50000 row(s) unloaded.
>>
@@ -1962,17 +1963,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>INTO '/user/trafodion/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:52:48.844
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:52:48.846
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
-Task: VERIFY SNAPSHO Status: Started Time: 2017-05-01 18:52:48.846
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:20.874
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:20.878
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.003
+Task: VERIFY SNAPSHO Status: Started Time: 2017-08-15 15:02:20.878
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended Time: 2017-05-01 18:52:49.204
-Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.358
-Task: EXTRACT Status: Started Time: 2017-05-01 18:52:49.204
+Task: VERIFY SNAPSHO Status: Ended Time: 2017-08-15 15:02:21.318
+Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.440
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:21.318
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:52:57.416
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:08.211
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:30.925
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:08.774
--- 20000 row(s) unloaded.
>>
@@ -2019,21 +2020,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/trafodion/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:52:59.366
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:52:59.377
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.011
-Task: CREATE SNAPSHO Status: Started Time: 2017-05-01 18:52:59.377
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:32.124
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:32.137
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.014
+Task: CREATE SNAPSHO Status: Started Time: 2017-08-15 15:02:32.138
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended Time: 2017-05-01 18:53:00.390
-Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:01.013
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:00.393
+Task: CREATE SNAPSHO Status: Ended Time: 2017-08-15 15:02:33.364
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:01.227
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:33.364
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:01.609
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.216
-Task: DELETE SNAPSHO Status: Started Time: 2017-05-01 18:53:01.609
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:34.688
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.323
+Task: DELETE SNAPSHO Status: Started Time: 2017-08-15 15:02:34.688
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended Time: 2017-05-01 18:53:01.627
-Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.018
+Task: DELETE SNAPSHO Status: Ended Time: 2017-08-15 15:02:34.720
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.032
--- 20000 row(s) unloaded.
>>
@@ -2041,7 +2042,7 @@ Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.018
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1493664777, failedModTS = 1493664780, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1502809349, failedModTS = 1502809353, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
(EXPR)
--------------------
@@ -2084,21 +2085,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/trafodion/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:53:04.714
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:53:04.726
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.012
-Task: CREATE SNAPSHO Status: Started Time: 2017-05-01 18:53:04.726
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:37.723
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:37.733
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
+Task: CREATE SNAPSHO Status: Started Time: 2017-08-15 15:02:37.733
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended Time: 2017-05-01 18:53:06.337
-Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:01.611
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:06.337
+Task: CREATE SNAPSHO Status: Ended Time: 2017-08-15 15:02:38.443
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.710
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:38.443
Rows Processed: 20000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:07.193
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.856
-Task: DELETE SNAPSHO Status: Started Time: 2017-05-01 18:53:07.193
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:40.918
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.648
+Task: DELETE SNAPSHO Status: Started Time: 2017-08-15 15:02:40.919
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended Time: 2017-05-01 18:53:07.204
-Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.011
+Task: DELETE SNAPSHO Status: Ended Time: 2017-08-15 15:02:40.989
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.007
--- 20000 row(s) unloaded.
>>
@@ -2114,7 +2115,7 @@ Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.011
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1493664781, failedModTS = 1493664786, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1502809354, failedModTS = 1502809358, failedLoc = hdfs://localhost:25600/user/trafodion/hive/exttables/unload_customer_demographics
CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_ESTIMATE CD_CREDIT_RATING CD_DEP_COUNT CD_DEP_EMPLOYED_COUNT CD_DEP_COLLEGE_COUNT
----------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -------------------- ---------------------------------------------------------------------------------------------------- ------------ --------------------- --------------------
@@ -2150,21 +2151,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/trafodion/hive/exttables/unload_customer_address'
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:53:10.415
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:53:10.420
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
-Task: CREATE SNAPSHO Status: Started Time: 2017-05-01 18:53:10.420
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:43.246
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:43.252
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: CREATE SNAPSHO Status: Started Time: 2017-08-15 15:02:43.252
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended Time: 2017-05-01 18:53:11.649
-Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:01.228
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:11.649
+Task: CREATE SNAPSHO Status: Ended Time: 2017-08-15 15:02:44.110
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.858
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:44.110
Rows Processed: 1998
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:12.226
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.577
-Task: DELETE SNAPSHO Status: Started Time: 2017-05-01 18:53:12.226
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:44.980
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.870
+Task: DELETE SNAPSHO Status: Started Time: 2017-08-15 15:02:44.980
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended Time: 2017-05-01 18:53:12.234
-Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.008
+Task: DELETE SNAPSHO Status: Ended Time: 2017-08-15 15:02:44.986
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.006
--- 1998 row(s) unloaded.
>>
@@ -2238,21 +2239,21 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/trafodion/hive/exttables/unload_customer_and_address'
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:53:14.484
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:53:14.491
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:47.233
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:47.240
Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
-Task: CREATE SNAPSHO Status: Started Time: 2017-05-01 18:53:14.491
+Task: CREATE SNAPSHO Status: Started Time: 2017-08-15 15:02:47.240
Snapshots created: 2
-Task: CREATE SNAPSHO Status: Ended Time: 2017-05-01 18:53:17.824
-Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:03.333
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:17.824
+Task: CREATE SNAPSHO Status: Ended Time: 2017-08-15 15:02:50.272
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:03.032
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:50.272
Rows Processed: 100000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:22.629
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:04.176
-Task: DELETE SNAPSHO Status: Started Time: 2017-05-01 18:53:22.736
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:02:54.174
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:03.902
+Task: DELETE SNAPSHO Status: Started Time: 2017-08-15 15:02:54.174
Snapshots deleted: 2
-Task: DELETE SNAPSHO Status: Ended Time: 2017-05-01 18:53:22.197
-Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.019
+Task: DELETE SNAPSHO Status: Ended Time: 2017-08-15 15:02:54.186
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.012
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -2319,21 +2320,21 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>INTO '/user/trafodion/hive/exttables/unload_customer_name'
+>select c_first_name,c_last_name from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started Time: 2017-05-01 18:53:25.364
-Task: EMPTY TARGET Status: Ended Time: 2017-05-01 18:53:25.366
-Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
-Task: CREATE SNAPSHO Status: Started Time: 2017-05-01 18:53:25.366
+Task: EMPTY TARGET Status: Started Time: 2017-08-15 15:02:58.169
+Task: EMPTY TARGET Status: Ended Time: 2017-08-15 15:02:58.257
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
+Task: CREATE SNAPSHO Status: Started Time: 2017-08-15 15:02:58.257
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended Time: 2017-05-01 18:53:26.197
-Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.653
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:26.198
+Task: CREATE SNAPSHO Status: Ended Time: 2017-08-15 15:02:58.684
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.659
+Task: EXTRACT Status: Started Time: 2017-08-15 15:02:58.684
Rows Processed: 100000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:27.567
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.548
-Task: DELETE SNAPSHO Status: Started Time: 2017-05-01 18:53:27.567
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:03:00.257
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.573
+Task: DELETE SNAPSHO Status: Started Time: 2017-08-15 15:03:00.257
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended Time: 2017-05-01 18:53:27.576
-Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.009
+Task: DELETE SNAPSHO Status: Ended Time: 2017-08-15 15:03:00.286
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.029
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -2377,16 +2378,16 @@ C_FIRST_NAME
>>--unload 100 --should give error [8447]
>>unload into '//\a//c' select * from CUSTOMER_ADDRESS;
Task: UNLOAD Status: Started
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:31.145
+Task: EXTRACT Status: Started Time: 2017-08-15 15:03:02.997
*** ERROR[8447] An error occurred during hdfs access. Error Detail: Java exception in hdfsCreate(). java.io.IOException: No FileSystem for scheme: null
-org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2676)
-org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2690)
-org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
-org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2733)
-org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2715)
-org.apache.hadoop.fs.FileSystem.get(FileSystem.java:382)
-org.trafodion.sql.SequenceFileWriter.hdfsCreate(SequenceFileWriter.java:155)
+org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2584)
+org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2591)
+org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
+org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2630)
+org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2612)
+org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370)
+org.trafodion.sql.SequenceFileWriter.hdfsCreate(SequenceFileWriter.java:156)
--- 0 row(s) unloaded.
>>
@@ -2411,10 +2412,10 @@ unload with delimiter 0 into '/user/trafodion/bulkload/test' select * from CUST
>>--unload 103 -- should not give an error
>>unload with delimiter '\a' into '/user/trafodion/bulkload/test' select * from customer_address;
Task: UNLOAD Status: Started
-Task: EXTRACT Status: Started Time: 2017-05-01 18:53:31.774
+Task: EXTRACT Status: Started Time: 2017-08-15 15:03:03.456
Rows Processed: 50000
-Task: EXTRACT Status: Ended Time: 2017-05-01 18:53:32.634
-Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.860
+Task: EXTRACT Status: Ended Time: 2017-08-15 15:03:04.542
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.086
--- 50000 row(s) unloaded.
>>--unload 24 -- should give an error