You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2018/12/05 21:54:48 UTC
[5/8] trafodion git commit: [TRAFODION-3234] Refactor hive meta calls
to be less resource intensive to support hive partitions if needed
[TRAFODION-3234] Refactor hive meta calls to be less resource intensive to support hive partitions if needed
Fixes for hive/TEST005 failure.
This also made some of the earlier failed test cases in this test suite to return correct results.
Data modification check was unnecessarily done in the scan operator. This has been removed.
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/7be2672d
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/7be2672d
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/7be2672d
Branch: refs/heads/master
Commit: 7be2672d6fcad8e48d7508ff8a04b51bbde48441
Parents: 7633327
Author: selvaganesang <se...@esgyn.com>
Authored: Tue Dec 4 15:09:25 2018 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Tue Dec 4 15:09:25 2018 +0000
----------------------------------------------------------------------
core/sql/executor/ExHdfsScan.cpp | 99 +++----------------------
core/sql/executor/hiveHook.cpp | 10 ++-
core/sql/optimizer/HDFSHook.cpp | 67 +++++++----------
core/sql/regress/hive/EXPECTED005 | 128 +++++++++++++++------------------
core/sql/regress/hive/TEST005 | 4 --
5 files changed, 98 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index c49a6a0..c4318d6 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -446,11 +446,12 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
step_ = ASSIGN_RANGES_AT_RUNTIME;
break;
}
- else if (getHdfsFileInfoListAsArray().isEmpty())
- {
- step_ = CHECK_FOR_DATA_MOD_AND_DONE;
- break;
- }
+ else {
+ if (useLibhdfsScan_)
+ step_ = INIT_HDFS_CURSOR;
+ else
+ step_ = SETUP_HDFS_SCAN;
+ }
beginRangeNum_ =
*(Lng32*)hdfsScanTdb().getHdfsFileRangeBeginList()->get(myInstNum_);
@@ -459,11 +460,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
*(Lng32*)hdfsScanTdb().getHdfsFileRangeNumList()->get(myInstNum_);
currRangeNum_ = beginRangeNum_;
-
- if (numRanges_ > 0)
- step_ = CHECK_FOR_DATA_MOD;
- else
- step_ = CHECK_FOR_DATA_MOD_AND_DONE;
+ if (numRanges_ <= 0)
+ step_ = DONE;
}
break;
@@ -479,85 +477,6 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
else
step_ = DONE;
break;
-
- case CHECK_FOR_DATA_MOD:
- case CHECK_FOR_DATA_MOD_AND_DONE:
- {
- char * dirPath = hdfsScanTdb().hdfsRootDir_;
- Int64 modTS = hdfsScanTdb().modTSforDir_;
- if ((dirPath == NULL) || (modTS == -1))
- dataModCheckDone_ = TRUE;
-
- if (NOT dataModCheckDone_)
- {
- dataModCheckDone_ = TRUE;
-
- Lng32 numOfPartLevels = hdfsScanTdb().numOfPartCols_;
-
- if (hdfsScanTdb().hdfsDirsToCheck())
- {
- // TBD
- }
-
- Int64 failedModTS = -1;
- Lng32 failedLocBufLen = 1000;
- char failedLocBuf[failedLocBufLen];
- retcode = ExpLOBinterfaceDataModCheck
- (lobGlob_,
- dirPath,
- hdfsScanTdb().hostName_,
- hdfsScanTdb().port_,
- modTS,
- numOfPartLevels,
- failedModTS,
- failedLocBuf, failedLocBufLen);
-
- if (retcode < 0)
- {
- Lng32 cliError = 0;
-
- Lng32 intParam1 = -retcode;
- ComDiagsArea * diagsArea = NULL;
- ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE),
- NULL, &intParam1,
- &cliError,
- NULL,
- "HDFS",
- (char*)"ExpLOBInterfaceDataModCheck",
- getLobErrStr(intParam1));
- pentry_down->setDiagsArea(diagsArea);
- step_ = HANDLE_ERROR_AND_DONE;
- break;
- }
-
- if (retcode == 1) // check failed
- {
- char errStr[200];
- str_sprintf(errStr, "genModTS = %ld, failedModTS = %ld",
- modTS, failedModTS);
-
- ComDiagsArea * diagsArea = NULL;
- ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR), NULL,
- NULL, NULL, NULL,
- errStr);
- pentry_down->setDiagsArea(diagsArea);
- step_ = HANDLE_ERROR_AND_DONE;
- break;
- }
- }
-
- if (step_ == CHECK_FOR_DATA_MOD_AND_DONE)
- step_ = DONE;
- else {
- if (useLibhdfsScan_)
- step_ = INIT_HDFS_CURSOR;
- else
- step_ = SETUP_HDFS_SCAN;
- }
- }
- break;
case SETUP_HDFS_SCAN:
{
if (hdfsScan_ != NULL)
@@ -810,7 +729,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
}
else
ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(EXE_HIVE_DATA_MOD_CHECK_ERROR));
+ (ExeErrorCode)(EXE_ERROR_FROM_LOB_INTERFACE));
pentry_down->setDiagsArea(diagsArea);
step_ = HANDLE_ERROR_AND_DONE;
break;
http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/executor/hiveHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/hiveHook.cpp b/core/sql/executor/hiveHook.cpp
index 284ba5c..234249a 100644
--- a/core/sql/executor/hiveHook.cpp
+++ b/core/sql/executor/hiveHook.cpp
@@ -307,8 +307,8 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc( const char* schemaName,
while (ptr) {
- if ( !(strcmp(ptr->tblName_, tblName)
- ||strcmp(ptr->schName_, schemaName))) {
+ if ( !(stricmp(ptr->tblName_, tblName)
+ ||stricmp(ptr->schName_, schemaName))) {
if ((NOT rereadFromMD) && (validate(ptr)))
return ptr;
else {
@@ -332,6 +332,9 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc( const char* schemaName,
ptr = ptr->next_;
}
+
+ if (validateOnly)
+ return NULL;
HVC_RetCode hvcRetcode;
HiveClient_JNI *hiveClient = HiveClient_JNI::newInstance(heap_, hvcRetcode);
@@ -569,7 +572,8 @@ Int32 hive_tbl_desc::getSortColNum(const char* name)
Int64 hive_tbl_desc::redeftime()
{
- Int64 result = creationTS_;
+ // creationTS_ is in seconds
+ Int64 result = creationTS_ * 1000;
if (redefineTS_ != -1)
result = redefineTS_;
return result;
http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/HDFSHook.cpp b/core/sql/optimizer/HDFSHook.cpp
index ed9b2fd..cc2bab5 100644
--- a/core/sql/optimizer/HDFSHook.cpp
+++ b/core/sql/optimizer/HDFSHook.cpp
@@ -854,50 +854,40 @@ NABoolean HHDFSTableStats::populate(struct hive_tbl_desc *htd)
Int32 hdfsPort = -1;
NAString tableDir;
- if (hsd)
- {
- if (hsd->isTextFile())
+ if (hsd) {
+ if (hsd->isTextFile())
type_ = TEXT_;
- else if (hsd->isSequenceFile())
+ else if (hsd->isSequenceFile())
type_ = SEQUENCE_;
- else if (hsd->isOrcFile())
+ else if (hsd->isOrcFile())
type_ = ORC_;
- else
+ else
type_ = UNKNOWN_;
- }
-
- while (hsd && diags_.isSuccess())
- {
- // split table URL into host, port and filename
- if (! splitLocation(hsd->location_,
- hdfsHost,
- hdfsPort,
- tableDir,
- diags_,
- hdfsPortOverride_)) {
- return FALSE;
- }
-
- if (! connectHDFS(hdfsHost, hdfsPort)) {
- return FALSE; // diags_ is set
- }
-
- // put back fully qualified URI
- tableDir = hsd->location_;
-
- // get the fine-resolution timestamp before visiting
- // the tree, to avoid losing any updates while this
- // method is executing
- computeModificationTSmsec();
-
- if (diags_.isSuccess())
+ }
+ // split table URL into host, port and filename
+ if (! splitLocation(hsd->location_,
+ hdfsHost,
+ hdfsPort,
+ tableDir,
+ diags_,
+ hdfsPortOverride_))
+ return FALSE;
+ if (! connectHDFS(hdfsHost, hdfsPort))
+ return FALSE; // diags_ is set
+ // put back fully qualified URI
+ tableDir = hsd->location_;
+ computeModificationTSmsec();
+ if (diags_.isSuccess()) {
+ htd->setRedeftime(modificationTSInMillisec_);
+ while (hsd && diags_.isSuccess()) {
// visit the directory
- processDirectory(tableDir, hsd->buckets_,
+ processDirectory(hsd->location_, hsd->buckets_,
hsd->isTrulyText(),
hsd->getRecordTerminator());
- hsd = hsd->next_;
- }
+ hsd = hsd->next_;
+ }
+ }
disconnectHDFS();
validationJTimestamp_ = JULIANTIMESTAMP();
@@ -1147,8 +1137,6 @@ void HHDFSTableStats::disconnectHDFS()
void HHDFSTableStats::computeModificationTSmsec()
{
- if (modificationTSInMillisec_ <= 0)
- {
HDFS_Client_RetCode rc;
// get a millisecond-resolution timestamp via JNI
@@ -1167,9 +1155,6 @@ void HHDFSTableStats::computeModificationTSmsec()
diags_.recordError(errMsg, "HHDFSTableStats::computeModificationTSmsec");
modificationTSInMillisec_ = -1;
}
- }
-
- return;
}
OsimHHDFSStatsBase* HHDFSTableStats::osimSnapShot(NAMemory * heap)
http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index c3f6a7b..d9475d8 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -72,7 +72,7 @@
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542230997205, failedModTS = 1542231173977, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900115861, failedModTS = 1543900283891, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
@@ -108,7 +108,7 @@ Y 9525
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542230997205, failedModTS = 1542231173977, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900115861, failedModTS = 1543900283891, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
@@ -118,8 +118,6 @@ N 9789
Y 9525
--- 3 row(s) selected.
->>-- because we don't invalidate in the executor,
->>-- this should still return 0 rows
>>
>>execute s2;
@@ -151,7 +149,7 @@ Y 9525
(EXPR)
--------------------
- 0
+ 9789
--- 1 row(s) selected.
>>-- although this should get an NATable cache
@@ -177,7 +175,7 @@ Y 9525
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231178414, failedModTS = 1542231182305, failedLoc = hdfs://localhost:9000/user/hive/warehouse/newtable
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900288480, failedModTS = 1543900294265, failedLoc = hdfs://localhost:9000/user/hive/warehouse/newtable
A
-------------------------
@@ -227,7 +225,7 @@ xyz
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231173977, failedModTS = 1542231203852, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900283891, failedModTS = 1543900317574, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
@@ -237,12 +235,11 @@ N 19631
Y 18984
--- 3 row(s) selected.
->>-- s1 should still return 0 rows - for now
>>execute s2;
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231173977, failedModTS = 1542231203852, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900283891, failedModTS = 1543900317574, failedLoc = hdfs://localhost:9000/user/trafodion/hive/exttables/customer_ddl
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
@@ -268,13 +265,12 @@ Y 18984
0
--- 1 row(s) selected.
->>-- s1 should still return 0 rows - for now
>>execute s2part;
(EXPR)
--------------------
- 0
+ 9789
--- 1 row(s) selected.
>>execute s3part;
@@ -282,7 +278,7 @@ Y 18984
(EXPR)
--------------------
- 0
+ 19314
--- 1 row(s) selected.
>>
@@ -298,7 +294,7 @@ Y 18984
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231200755, failedModTS = 1542231206302, failedLoc = hdfs://localhost:9000/user/hive/warehouse/newtable
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900313695, failedModTS = 1543900320754, failedLoc = hdfs://localhost:9000/user/hive/warehouse/newtable
A B
----------- -------------------------
@@ -354,7 +350,7 @@ Y 18984
(EXPR)
--------------------
- 0
+ 9789
--- 1 row(s) selected.
>>execute s4part;
@@ -362,7 +358,7 @@ Y 18984
(EXPR)
--------------------
- 0
+ 19999
--- 1 row(s) selected.
>>select count(*) from tbl_utf8;
@@ -631,18 +627,18 @@ C1 C2 C3 C4 C
--- 1 row(s) selected.
>>load with continue on error into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task: CLEANUP Status: Started Time: 2018-11-14 21:34:43.60760
-Task: CLEANUP Status: Ended Time: 2018-11-14 21:34:43.76350
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.016
-Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:43.76422
+Task: CLEANUP Status: Started Time: 2018-12-04 05:13:10.600264
+Task: CLEANUP Status: Ended Time: 2018-12-04 05:13:10.612952
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013
+Task: LOADING DATA Status: Started Time: 2018-12-04 05:13:10.613035
Rows Processed: 8
Error Rows: 5
-Task: LOADING DATA Status: Ended Time: 2018-11-14 21:34:43.275300
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.199
-Task: COMPLETION Status: Started Time: 2018-11-14 21:34:43.275441
+Task: LOADING DATA Status: Ended Time: 2018-12-04 05:13:10.766853
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.154
+Task: COMPLETION Status: Started Time: 2018-12-04 05:13:10.766892
Rows Loaded: 3
-Task: COMPLETION Status: Ended Time: 2018-11-14 21:34:44.92332
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.817
+Task: COMPLETION Status: Ended Time: 2018-12-04 05:13:11.231800
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.465
--- 3 row(s) loaded.
>>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -658,19 +654,19 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.817
--- 3 row(s) deleted.
>>load with log error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task: CLEANUP Status: Started Time: 2018-11-14 21:34:45.188714
-Task: CLEANUP Status: Ended Time: 2018-11-14 21:34:45.202159
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013
- Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213445
-Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:45.202236
+Task: CLEANUP Status: Started Time: 2018-12-04 05:13:12.257338
+Task: CLEANUP Status: Ended Time: 2018-12-04 05:13:12.272165
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.015
+ Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051312
+Task: LOADING DATA Status: Started Time: 2018-12-04 05:13:12.272230
Rows Processed: 8
Error Rows: 5
-Task: LOADING DATA Status: Ended Time: 2018-11-14 21:34:45.474416
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.272
-Task: COMPLETION Status: Started Time: 2018-11-14 21:34:45.474451
+Task: LOADING DATA Status: Ended Time: 2018-12-04 05:13:12.508271
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.236
+Task: COMPLETION Status: Started Time: 2018-12-04 05:13:12.508318
Rows Loaded: 3
-Task: COMPLETION Status: Ended Time: 2018-11-14 21:34:46.101985
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.628
+Task: COMPLETION Status: Ended Time: 2018-12-04 05:13:13.152197
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.644
--- 3 row(s) loaded.
>>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -683,19 +679,19 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.628
--- 1 row(s) selected.
>>load with log error rows to '/user/trafodion/bulkload/logs/TEST005' into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task: CLEANUP Status: Started Time: 2018-11-14 21:34:47.154307
-Task: CLEANUP Status: Ended Time: 2018-11-14 21:34:47.162209
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.008
- Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213447
-Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:47.162253
+Task: CLEANUP Status: Started Time: 2018-12-04 05:13:14.138305
+Task: CLEANUP Status: Ended Time: 2018-12-04 05:13:14.142425
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.004
+ Logging Location: /user/trafodion/bulkload/logs/TEST005/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051314
+Task: LOADING DATA Status: Started Time: 2018-12-04 05:13:14.142474
Rows Processed: 8
Error Rows: 5
-Task: LOADING DATA Status: Ended Time: 2018-11-14 21:34:47.473145
-Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.311
-Task: COMPLETION Status: Started Time: 2018-11-14 21:34:47.473192
+Task: LOADING DATA Status: Ended Time: 2018-12-04 05:13:14.379124
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.237
+Task: COMPLETION Status: Started Time: 2018-12-04 05:13:14.379186
Rows Loaded: 3
-Task: COMPLETION Status: Ended Time: 2018-11-14 21:34:48.262413
-Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.789
+Task: COMPLETION Status: Ended Time: 2018-12-04 05:13:14.991405
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.612
--- 3 row(s) loaded.
>>select count(*) from trafodion.seabase.traf_tbl_bad;
@@ -711,10 +707,10 @@ Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.789
--- 6 row(s) deleted.
>>load with stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task: CLEANUP Status: Started Time: 2018-11-14 21:34:49.353872
-Task: CLEANUP Status: Ended Time: 2018-11-14 21:34:49.366515
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.013
-Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:49.366571
+Task: CLEANUP Status: Started Time: 2018-12-04 05:13:16.65887
+Task: CLEANUP Status: Ended Time: 2018-12-04 05:13:16.72797
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.007
+Task: LOADING DATA Status: Started Time: 2018-12-04 05:13:16.72836
*** ERROR[8113] The maximum number of error rows is exceeded.
@@ -729,11 +725,11 @@ Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:49.366571
--- 1 row(s) selected.
>>load with log error rows, stop after 3 error rows into trafodion.seabase.traf_tbl_bad select * from tbl_bad;
Task: LOAD Status: Started Object: TRAFODION.SEABASE.TRAF_TBL_BAD
-Task: CLEANUP Status: Started Time: 2018-11-14 21:34:50.532300
-Task: CLEANUP Status: Ended Time: 2018-11-14 21:34:50.542271
-Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.010
- Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181114_213450
-Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:50.542323
+Task: CLEANUP Status: Started Time: 2018-12-04 05:13:17.218725
+Task: CLEANUP Status: Ended Time: 2018-12-04 05:13:17.234042
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.015
+ Logging Location: /user/trafodion/bulkload/logs/ERR_TRAFODION.SEABASE.TRAF_TBL_BAD_20181204_051317
+Task: LOADING DATA Status: Started Time: 2018-12-04 05:13:17.234092
*** ERROR[8113] The maximum number of error rows is exceeded.
@@ -823,7 +819,7 @@ Task: LOADING DATA Status: Started Time: 2018-11-14 21:34:50.542323
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231316268, failedModTS = 1542231318359, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900418287, failedModTS = 1543900423606, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
A
-----------
@@ -882,7 +878,7 @@ A B
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231324672, failedModTS = 1542231325768, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900429674, failedModTS = 1543900430804, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
--- 0 row(s) selected.
>>insert into hive.hive.thive values (10, 20);
@@ -906,7 +902,7 @@ A B
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231326748, failedModTS = 1542231328569, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1543900431824, failedModTS = 1543900433636, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive
--- 0 row(s) selected.
>>
@@ -995,7 +991,7 @@ t005part.a t005part.b t005part.c
>>invoke hive.hive.thive_insert_smallint;
-- Definition of hive table HIVE.HIVE.THIVE_INSERT_SMALLINT
--- Definition current Wed Nov 14 21:39:06 2018
+-- Definition current Tue Dec 4 05:17:20 2018
(
A SMALLINT
@@ -1016,7 +1012,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT
/* Trafodion DDL */
REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT;
-/* ObjectUID = 4263396855838728432 */
+/* ObjectUID = 4414832062427972126 */
--- SQL operation complete.
>>
@@ -1029,10 +1025,6 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_SMALLINT;
>>insert into hive.hive.thive_insert_smallint select * from
+> (values (10), (11111111), (21), (22222222));
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231120366, failedModTS = 1542231546673, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive_insert_smallint
-
--- 4 row(s) inserted.
>>select * from hive.hive.thive_insert_smallint;
@@ -1070,10 +1062,6 @@ A
>>insert into hive.hive.thive_insert_smallint select * from
+> (values (10), (11111111), (21), (22222222));
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231548908, failedModTS = 1542231550707, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive_insert_smallint
-
--- 2 row(s) inserted.
>>select * from hive.hive.thive_insert_smallint;
@@ -1113,7 +1101,7 @@ A
>>invoke hive.hive.thive_insert_varchar;
-- Definition of hive table HIVE.HIVE.THIVE_INSERT_VARCHAR
--- Definition current Wed Nov 14 21:39:15 2018
+-- Definition current Tue Dec 4 05:17:29 2018
(
A VARCHAR(1 CHAR) CHARACTER SET UTF8 COLLATE
@@ -1138,7 +1126,7 @@ CREATE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR
/* Trafodion DDL */
REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR;
-/* ObjectUID = 4263396855838729348 */
+/* ObjectUID = 4414832062427972695 */
--- SQL operation complete.
>>cqd hive_insert_error_mode '1';
@@ -1149,10 +1137,6 @@ REGISTER /*INTERNAL*/ HIVE TABLE HIVE.HIVE.THIVE_INSERT_VARCHAR;
--- SQL operation complete.
>>insert into hive.hive.thive_insert_varchar values ('abcddcba','efghijkl');
-*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1542231129568, failedModTS = 1542231555639, failedLoc = hdfs://localhost:9000/user/hive/warehouse/thive_insert_varchar
-
--- 1 row(s) inserted.
>>
>>cqd hive_max_string_length_in_bytes '20';
http://git-wip-us.apache.org/repos/asf/trafodion/blob/7be2672d/core/sql/regress/hive/TEST005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005 b/core/sql/regress/hive/TEST005
index 6521786..bf87a69 100644
--- a/core/sql/regress/hive/TEST005
+++ b/core/sql/regress/hive/TEST005
@@ -122,8 +122,6 @@ prepare s2part from
--order by 1
;
execute s1;
--- because we don't invalidate in the executor,
--- this should still return 0 rows
execute s2;
-- should get an NATable cache
@@ -173,11 +171,9 @@ prepare s3part from
--order by 1
;
execute s1;
--- s1 should still return 0 rows - for now
execute s2;
execute s3;
execute s1part;
--- s1 should still return 0 rows - for now
execute s2part;
execute s3part;