You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by su...@apache.org on 2016/08/07 23:48:06 UTC
[1/5] incubator-trafodion git commit: [TRAFODION-2138] Hive scan on
wide tables can result in lost rows or error
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 9f3542dcf -> a72306e4e
[TRAFODION-2138] Hive scan on wide tables can result in lost rows or error
The fix has three parts
a) RangeTailIOSize will now default to maximum length of a row, if > 16KB.
Each esp range will look ahead by this size. Previously this was hard coded to 16KB.
b) If a whole buffer does not have the start of a row, that is not an error.
c) We raise an error during compile if maximum row size is greater than size of IO buffer.
We need to have the whole row (from start to finish) in a contiguous buffer to do the rest of our logic.
Currently IO buffer defaults to 64 MB.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/6d266221
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/6d266221
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/6d266221
Branch: refs/heads/master
Commit: 6d266221b959c14319a760716939f261a2460276
Parents: 98b00b6
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Aug 3 20:35:43 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Aug 3 20:35:43 2016 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 1 +
core/sql/executor/ExHdfsScan.cpp | 52 +++++++++-----
core/sql/exp/ExpLOBaccess.cpp | 2 +-
core/sql/generator/GenRelScan.cpp | 10 ++-
core/sql/optimizer/BindRelExpr.cpp | 18 +++++
core/sql/regress/seabase/EXPECTED031 | 108 +++++++++++++++++++++++++-----
core/sql/regress/seabase/TEST031 | 19 ++++++
core/sql/sqlcomp/nadefaults.cpp | 3 +-
8 files changed, 179 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt
index 88ea933..b5704f8 100644
--- a/core/sql/bin/SqlciErrors.txt
+++ b/core/sql/bin/SqlciErrors.txt
@@ -1282,6 +1282,7 @@ $1~String1 --------------------------------
4223 42000 99999 BEGINNER MAJOR DBADMIN $0~String0 not supported in this software version.
4224 42000 99999 BEGINNER MAJOR DBADMIN The directory $0~String0 is not a valid Hive location.
4225 42000 99999 BEGINNER MAJOR DBADMIN Number of column families cannot exceed 32.
+4226 42000 99999 BEGINNER MAJOR DBADMIN Table $0~TableName has a maximum record length of $0~Int0 which is greater than HDFS_IO_BUFFERSIZE. Increase buffer size setting, or reduce HIVE_MAX_STRING_LENGTH.
4240 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Expression $0~string0 in the DIVISION BY clause references columns other than clustering key columns.
4241 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The value $0~string0 is not supported at this place in the DIVISION BY clause, only constants are allowed.
4242 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The value $0~string0 is not supported at this place in the DIVISION BY clause, only key columns are allowed.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 7f4cf90..978bd41 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -563,7 +563,8 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
}
else
{
-
+ Int64 rangeTail = hdfo_->fileIsSplitEnd() ?
+ hdfsScanTdb().rangeTailIOSize_ : 0;
openType = 2; // must open
retcode = ExpLOBInterfaceSelectCursor
(lobGlob_,
@@ -573,7 +574,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
hdfsScanTdb().hostName_,
hdfsScanTdb().port_,
0, NULL, // handle not valid for non lob access
- bytesLeft_, // max bytes
+ bytesLeft_ + rangeTail, // max bytes
cursorId_,
requestTag_, Lob_Memory,
@@ -608,6 +609,9 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
hdfsFileName_ = hdfo->fileName();
sprintf(cursorId, "%d", currRangeNum_ + 1);
+ rangeTail = hdfo->fileIsSplitEnd() ?
+ hdfsScanTdb().rangeTailIOSize_ : 0;
+
openType = 1; // preOpen
retcode = ExpLOBInterfaceSelectCursor
@@ -618,7 +622,7 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
hdfsScanTdb().hostName_,
hdfsScanTdb().port_,
0, NULL,//handle not relevant for non lob access
- hdfo->getBytesToRead(), // max bytes
+ hdfo->getBytesToRead() + rangeTail, // max bytes
cursorId,
requestTag_, Lob_Memory,
@@ -812,23 +816,39 @@ ExWorkProcRetcode ExHdfsScanTcb::work()
{
// Position in the hdfsScanBuffer_ to the
// first record delimiter.
- hdfsBufNextRow_ = hdfs_strchr(hdfsScanBuffer_,
- hdfsScanTdb().recordDelimiter_, hdfsScanBuffer_+trailingPrevRead_+ bytesRead_, checkRangeDelimiter_, hdfsScanTdb().getHiveScanMode(), &changedLen);
+ hdfsBufNextRow_ =
+ hdfs_strchr(hdfsScanBuffer_,
+ hdfsScanTdb().recordDelimiter_,
+ hdfsScanBuffer_+trailingPrevRead_+
+ min(bytesRead_, hdfo_->bytesToRead_),
+ checkRangeDelimiter_,
+ hdfsScanTdb().getHiveScanMode(), &changedLen);
// May be that the record is too long? Or data isn't ascii?
// Or delimiter is incorrect.
if (! hdfsBufNextRow_)
{
- ComDiagsArea *diagsArea = NULL;
-
- ExRaiseSqlError(getHeap(), &diagsArea,
- (ExeErrorCode)(8446), NULL,
- NULL, NULL, NULL,
- (char*)"No record delimiter found in buffer from hdfsRead.",
- NULL);
- // no need to log errors in this case (bulk load) since this is a major issue
- // and need to be correxted
- pentry_down->setDiagsArea(diagsArea);
- step_ = HANDLE_ERROR_WITH_CLOSE;
+ if (hdfo_->bytesToRead_ < hdfsScanTdb().rangeTailIOSize_)
+ {
+ // for wide rows it is not an error if a whole range
+ // does not include a record delimiter. RangeTaileIOSize
+ // is set to max row size in generator by default.
+ // It is also checked in the compiler that rowsize
+ // is less than buffer size.
+ step_ = CLOSE_HDFS_CURSOR;
+ }
+ else
+ {
+ ComDiagsArea *diagsArea = NULL;
+ ExRaiseSqlError(getHeap(), &diagsArea,
+ (ExeErrorCode)(8446), NULL,
+ NULL, NULL, NULL,
+ (char*)"No record delimiter found in buffer from hdfsRead.",
+ NULL);
+ // no need to log errors in this case (bulk load) since
+ // this is a major issue and needs to be corrected
+ pentry_down->setDiagsArea(diagsArea);
+ step_ = HANDLE_ERROR_WITH_CLOSE;
+ }
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/exp/ExpLOBaccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.cpp b/core/sql/exp/ExpLOBaccess.cpp
index 575bb92..773ff92 100644
--- a/core/sql/exp/ExpLOBaccess.cpp
+++ b/core/sql/exp/ExpLOBaccess.cpp
@@ -2800,7 +2800,7 @@ Ex_Lob_Error ExLobGlobals::performRequest(ExLobHdfsRequest *request)
buf->data_ = (char *) (getHeap())->allocateMemory( cursor->bufMaxSize_);
lobPtr->stats_.buffersUsed++;
}
- size = min(cursor->bufMaxSize_, (cursor->maxBytes_ - cursor->bytesRead_ + (16 * 1024)));
+ size = min(cursor->bufMaxSize_, (cursor->maxBytes_ - cursor->bytesRead_));
if (buf->data_) {
lobPtr->readCursorDataSimple(buf->data_, size, *cursor, buf->bytesRemaining_);
buf->bytesUsed_ = 0;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/generator/GenRelScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp
index 0b152ff..1ea3823 100644
--- a/core/sql/generator/GenRelScan.cpp
+++ b/core/sql/generator/GenRelScan.cpp
@@ -1173,7 +1173,6 @@ if (hTabStats->isOrcFile())
{
hdfsBufSize = (Int64)CmpCommon::getDefaultNumeric(HDFS_IO_BUFFERSIZE);
hdfsBufSize = hdfsBufSize * 1024; // convert to bytes
-
Int64 hdfsBufSizeTesting = (Int64)
CmpCommon::getDefaultNumeric(HDFS_IO_BUFFERSIZE_BYTES);
if (hdfsBufSizeTesting)
@@ -1182,6 +1181,15 @@ if (hTabStats->isOrcFile())
UInt32 rangeTailIOSize = (UInt32)
CmpCommon::getDefaultNumeric(HDFS_IO_RANGE_TAIL);
+ if (rangeTailIOSize == 0)
+ {
+ rangeTailIOSize = getTableDesc()->getNATable()->getRecordLength() +
+ (getTableDesc()->getNATable()->getClusteringIndex()->
+ getAllColumns().entries())*2;
+ // for each range we look ahead in the next range upto the maximum
+ // record length to find the end of record delimiter.
+ rangeTailIOSize = MAXOF(rangeTailIOSize, 16*1024);
+ }
char * tablename =
space->AllocateAndCopyToAlignedSpace(GenGetQualifiedName(getIndexDesc()->getNAFileSet()->getFileSetName()), 0);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index e2960f9..70c0b12 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -7582,6 +7582,24 @@ RelExpr *Scan::bindNode(BindWA *bindWA)
}
}
+ if (naTable->isHiveTable() &&
+ !(naTable->getClusteringIndex()->getHHDFSTableStats()->isOrcFile() ||
+ naTable->getClusteringIndex()->getHHDFSTableStats()
+ ->isSequenceFile()) &&
+ (CmpCommon::getDefaultNumeric(HDFS_IO_BUFFERSIZE_BYTES) == 0) &&
+ (naTable->getRecordLength() >
+ CmpCommon::getDefaultNumeric(HDFS_IO_BUFFERSIZE)*1024))
+ {
+ // do not raise error if buffersize is set though buffersize_bytes.
+ // Typically this setting is used for testing alone.
+ *CmpCommon::diags() << DgSqlCode(-4226)
+ << DgTableName(
+ naTable->getTableName().
+ getQualifiedNameAsAnsiString())
+ << DgInt0(naTable->getRecordLength());
+ bindWA->setErrStatus();
+ return NULL;
+ }
// Bind the base class.
//
RelExpr *boundExpr = bindSelf(bindWA);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/regress/seabase/EXPECTED031
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED031 b/core/sql/regress/seabase/EXPECTED031
index 873ed2c..c3141d2 100644
--- a/core/sql/regress/seabase/EXPECTED031
+++ b/core/sql/regress/seabase/EXPECTED031
@@ -431,19 +431,19 @@ Z Z (EXPR) (EXPR)
--- 1 row(s) selected.
>>select substring(a, 1, 10), cast(b as char(10)) from hive.hive.t031hive;
-(EXPR) (EXPR)
----------------------------------------- ----------------------------------------
-
-aaaaaaaaaa def
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-a zzz
-aaaaaaaaaa ?
-aaaaaaaaaa ?
-aaaaaaaaaa ?
+(EXPR) (EXPR)
+---------- ----------
+
+aaaaaaaaaa def
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+a zzz
+aaaaaaaaaa ?
+aaaaaaaaaa ?
+aaaaaaaaaa ?
--- 10 row(s) selected.
>>select [last 0] * from hive.hive.t031hive;
@@ -472,6 +472,84 @@ aaaaaaaaaa ?
--- 1 row(s) selected.
>>
+>>control query shape sort_groupby(exchange(cut));
+
+--- SQL operation complete.
+>>cqd hive_min_bytes_per_esp_partition '2000000' ;
+
+--- SQL operation complete.
+>>prepare s2 from select count(*) from hive.hive.t031hive;
+
+--- SQL command prepared.
+>>explain options 'f' s2 ;
+
+LC RC OP OPERATOR OPT DESCRIPTION CARD
+---- ---- ---- -------------------- -------- -------------------- ---------
+
+4 . 5 root 1.00E+000
+3 . 4 sort_partial_aggr_ro 1.00E+000
+2 . 3 esp_exchange 1:2(hash2) 1.00E+000
+1 . 2 sort_partial_aggr_le 1.00E+000
+. . 1 hive_scan T031HIVE 1.00E+000
+
+--- SQL operation complete.
+>>execute s2 ;
+
+(EXPR)
+--------------------
+
+ 10
+
+--- 1 row(s) selected.
+>>
+>>cqd hive_min_bytes_per_esp_partition '1000000' ;
+
+--- SQL operation complete.
+>>prepare s2 from select count(*) from hive.hive.t031hive;
+
+--- SQL command prepared.
+>>explain options 'f' s2 ;
+
+LC RC OP OPERATOR OPT DESCRIPTION CARD
+---- ---- ---- -------------------- -------- -------------------- ---------
+
+4 . 5 root 1.00E+000
+3 . 4 sort_partial_aggr_ro 1.00E+000
+2 . 3 esp_exchange 1:4(hash2) 1.00E+000
+1 . 2 sort_partial_aggr_le 1.00E+000
+. . 1 hive_scan T031HIVE 1.00E+000
+
+--- SQL operation complete.
+>>execute s2 ;
+
+(EXPR)
+--------------------
+
+ 10
+
+--- 1 row(s) selected.
+>>
+>>-- test error
+>>cqd hdfs_io_buffersize '3' ;
+
+--- SQL operation complete.
+>>prepare s2 from select count(*) from hive.hive.t031hive;
+
+*** ERROR[4226] Table HIVE.HIVE.T031HIVE has a maximum record length of 2000016 which is greater than HDFS_IO_BUFFERSIZE. Increase buffer size setting, or reduce HIVE_MAX_STRING_LENGTH.
+
+*** ERROR[8822] The statement was not prepared.
+
+>>
+>>control query shape cut ;
+
+--- SQL operation complete.
+>>cqd hdfs_io_buffersize reset ;
+
+--- SQL operation complete.
+>>cqd hive_min_bytes_per_esp_partition reset;
+
+--- SQL operation complete.
+>>
>>-- default USER
>>drop table if exists t031t1;
@@ -482,7 +560,7 @@ aaaaaaaaaa ?
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Wed Jul 13 23:30:01 2016
+-- Definition current Wed Aug 3 15:45:17 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -513,7 +591,7 @@ A B
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Wed Jul 13 23:30:02 2016
+-- Definition current Wed Aug 3 15:45:22 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/regress/seabase/TEST031
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/TEST031 b/core/sql/regress/seabase/TEST031
index 382e18f..da3dc51 100644
--- a/core/sql/regress/seabase/TEST031
+++ b/core/sql/regress/seabase/TEST031
@@ -166,6 +166,25 @@ delete from t031t1;
insert into t031t1 select * from hive.hive.t031hive;
select count(*) from t031t1;
+control query shape sort_groupby(exchange(cut));
+cqd hive_min_bytes_per_esp_partition '2000000' ;
+prepare s2 from select count(*) from hive.hive.t031hive;
+explain options 'f' s2 ;
+execute s2 ;
+
+cqd hive_min_bytes_per_esp_partition '1000000' ;
+prepare s2 from select count(*) from hive.hive.t031hive;
+explain options 'f' s2 ;
+execute s2 ;
+
+-- test error
+cqd hdfs_io_buffersize '3' ;
+prepare s2 from select count(*) from hive.hive.t031hive;
+
+control query shape cut ;
+cqd hdfs_io_buffersize reset ;
+cqd hive_min_bytes_per_esp_partition reset;
+
-- default USER
drop table if exists t031t1;
create table t031t1 (a int, b varchar(20) default user);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/6d266221/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 25c5672..4120de3 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -1796,7 +1796,8 @@ SDDkwd__(EXE_DIAGNOSTIC_EVENTS, "OFF"),
DDui1__(HDFS_IO_BUFFERSIZE, "65536"),
DDui___(HDFS_IO_BUFFERSIZE_BYTES, "0"),
- DDui1__(HDFS_IO_RANGE_TAIL, "16384"),
+ // The value 0 denotes RangeTail = max record length of table.
+ DDui___(HDFS_IO_RANGE_TAIL, "0"),
DDkwd__(HDFS_PREFETCH, "ON"),
DDkwd__(HDFS_READ_CONTINUE_ON_ERROR, "OFF"),
DDui1__(HDFS_REPLICATION, "1"),
[4/5] incubator-trafodion git commit: Testware change due to rework.
Posted by su...@apache.org.
Testware change due to rework.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f5a716d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f5a716d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f5a716d2
Branch: refs/heads/master
Commit: f5a716d23992f89c8f1f7578c850425ff74bcec4
Parents: bfd3cdb
Author: Suresh Subbiah <su...@apache.org>
Authored: Fri Aug 5 21:59:52 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Fri Aug 5 21:59:52 2016 +0000
----------------------------------------------------------------------
core/sql/regress/seabase/EXPECTED031 | 49 +++++++++++++++----------------
1 file changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/f5a716d2/core/sql/regress/seabase/EXPECTED031
----------------------------------------------------------------------
diff --git a/core/sql/regress/seabase/EXPECTED031 b/core/sql/regress/seabase/EXPECTED031
index 9c2255d..253b632 100644
--- a/core/sql/regress/seabase/EXPECTED031
+++ b/core/sql/regress/seabase/EXPECTED031
@@ -431,19 +431,19 @@ Z Z (EXPR) (EXPR)
--- 1 row(s) selected.
>>select substring(a, 1, 10), cast(b as char(10)) from hive.hive.t031hive;
-(EXPR) (EXPR)
----------- ----------
-
-aaaaaaaaaa def
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-aaaaaaaaaa zzz
-a zzz
-aaaaaaaaaa ?
-aaaaaaaaaa ?
-aaaaaaaaaa ?
+(EXPR) (EXPR)
+---------------------------------------- ----------------------------------------
+
+aaaaaaaaaa def
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+aaaaaaaaaa zzz
+a zzz
+aaaaaaaaaa ?
+aaaaaaaaaa ?
+aaaaaaaaaa ?
--- 10 row(s) selected.
>>select [last 0] * from hive.hive.t031hive;
@@ -535,7 +535,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
--- SQL operation complete.
>>prepare s2 from select count(*) from hive.hive.t031hive;
-*** ERROR[4226] Table HIVE.HIVE.T031HIVE has a maximum record length of 2000016 which is greater than HDFS_IO_BUFFERSIZE. Increase buffer size setting, or reduce HIVE_MAX_STRING_LENGTH.
+*** ERROR[4226] Table HIVE.HIVE.T031HIVE has a maximum record length of 2000016 which is greater than the HDFS_IO_BUFFERSIZE default. Increase buffer size setting, or reduce the HIVE_MAX_STRING_LENGTH default.
*** ERROR[8822] The statement was not prepared.
@@ -560,8 +560,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Wed Aug 3 15:45:17 2016
-
+-- Definition current Fri Aug 5 21:55:49 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -592,7 +591,7 @@ A B
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Wed Aug 3 15:45:22 2016
+-- Definition current Fri Aug 5 21:55:57 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -643,7 +642,7 @@ A B C
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Tue Aug 2 19:11:35 2016
+-- Definition current Fri Aug 5 21:56:03 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -672,7 +671,7 @@ A B C
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Tue Aug 2 19:11:42 2016
+-- Definition current Fri Aug 5 21:56:13 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -697,7 +696,7 @@ A B C
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Tue Aug 2 19:11:51 2016
+-- Definition current Fri Aug 5 21:56:16 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -717,7 +716,7 @@ A B C
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Tue Aug 2 19:11:53 2016
+-- Definition current Fri Aug 5 21:56:22 2016
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
@@ -732,7 +731,7 @@ A B C
>>invoke t031v1;
-- Definition of Trafodion view TRAFODION.SCH.T031V1
--- Definition current Tue Aug 2 19:11:54 2016
+-- Definition current Fri Aug 5 21:56:24 2016
(
A INT DEFAULT NULL
@@ -755,7 +754,7 @@ A B C
>>invoke t031v1;
-- Definition of Trafodion view TRAFODION.SCH.T031V1
--- Definition current Tue Aug 2 19:11:59 2016
+-- Definition current Fri Aug 5 21:56:31 2016
(
A INT DEFAULT NULL
@@ -773,7 +772,7 @@ A B C
>>invoke t031v1;
-- Definition of Trafodion view TRAFODION.SCH.T031V1
--- Definition current Tue Aug 2 19:12:01 2016
+-- Definition current Fri Aug 5 21:56:35 2016
(
A INT DEFAULT NULL
@@ -857,7 +856,7 @@ CREATE INDEX T031T1I1 ON TRAFODION.SCH.T031T1
>>invoke table(index_table t031t1i1);
-- Definition of Trafodion table TRAFODION.SCH.T031T1I1
--- Definition current Tue Aug 2 19:12:17 2016
+-- Definition current Fri Aug 5 21:56:56 2016
(
"A@" INT NO DEFAULT
[2/5] incubator-trafodion git commit: Merge remote branch 'origin'
into traf-2094
Posted by su...@apache.org.
Merge remote branch 'origin' into traf-2094
Resolving merge conflicts.
Conflicts:
core/sql/regress/seabase/EXPECTED031
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/66d03a83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/66d03a83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/66d03a83
Branch: refs/heads/master
Commit: 66d03a83489ec6d72b8d98f54a6530253eb2682b
Parents: 6d26622 2a0a301
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Aug 3 20:53:36 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Aug 3 20:53:36 2016 +0000
----------------------------------------------------------------------
core/sql/arkcmp/CmpStoredProc.cpp | 96 +-
core/sql/arkcmp/CmpStoredProc.h | 21 +-
core/sql/bin/SqlciErrors.txt | 3 +-
core/sql/common/CmpCommon.h | 1 +
core/sql/common/ComSmallDefs.h | 3 +-
core/sql/common/OperTypeEnum.h | 2 +
core/sql/generator/GenExplain.cpp | 99 +-
core/sql/generator/GenPreCode.cpp | 4 +-
core/sql/generator/GenRelExeUtil.cpp | 43 +-
core/sql/generator/GenRelMisc.cpp | 129 +-
core/sql/generator/GenRelScan.cpp | 41 +-
core/sql/generator/GenUdr.cpp | 4 +-
core/sql/generator/Generator.cpp | 782 ++++---
core/sql/generator/Generator.h | 78 +-
core/sql/nskgmake/sqlcat/Makefile | 1 +
core/sql/optimizer/BindItemExpr.cpp | 7 +-
core/sql/optimizer/BindRI.cpp | 28 +-
core/sql/optimizer/BindRelExpr.cpp | 97 +-
core/sql/optimizer/BindWA.h | 6 +-
core/sql/optimizer/EncodedKeyValue.cpp | 77 +-
core/sql/optimizer/EncodedKeyValue.h | 21 +-
core/sql/optimizer/ItemConstr.h | 5 +-
core/sql/optimizer/NAColumn.cpp | 192 +-
core/sql/optimizer/NAColumn.h | 4 +-
core/sql/optimizer/NAFileSet.cpp | 2 +-
core/sql/optimizer/NAFileSet.h | 10 +-
core/sql/optimizer/NARoutine.cpp | 66 +-
core/sql/optimizer/NARoutine.h | 2 +-
core/sql/optimizer/NATable.cpp | 2067 +++++++++---------
core/sql/optimizer/NATable.h | 14 +-
core/sql/optimizer/RelExeUtil.cpp | 24 +-
core/sql/optimizer/RelExeUtil.h | 20 +-
core/sql/optimizer/RelMisc.h | 2 +-
core/sql/optimizer/RelRoutine.cpp | 15 +-
core/sql/optimizer/RelRoutine.h | 28 +-
core/sql/optimizer/RelScan.h | 10 +-
core/sql/optimizer/RelStoredProc.cpp | 2 +-
core/sql/optimizer/ValueDesc.cpp | 7 +-
core/sql/parser/AllStmtDDL.h | 1 +
core/sql/parser/AllStmtDDLAlterTable.h | 1 +
core/sql/parser/BindStmtDDL.cpp | 31 +
core/sql/parser/ElemDDLNode.cpp | 12 +
core/sql/parser/ElemDDLNode.h | 4 +
core/sql/parser/StmtDDLAlter.cpp | 91 +
core/sql/parser/StmtDDLAlterSchema.h | 177 ++
core/sql/parser/StmtDDLAlterTableStoredDesc.h | 102 +
core/sql/parser/sqlparser.y | 80 +-
core/sql/regress/compGeneral/EXPECTED001.SB | 2 +
core/sql/regress/executor/EXPECTED013.SB | 2 +
core/sql/regress/executor/TEST122 | 6 +
core/sql/regress/fullstack2/EXPECTED062 | 6 +
core/sql/regress/fullstack2/EXPECTED062.RELEASE | 6 +
core/sql/regress/hive/EXPECTED018 | 187 +-
core/sql/regress/hive/TEST018 | 3 +
core/sql/regress/seabase/EXPECTED031 | 243 +-
core/sql/regress/seabase/TEST031 | 48 +-
core/sql/regress/tools/sbdefs | 2 +
core/sql/sqlcat/ReadTableDef.cpp | 404 ----
core/sql/sqlcat/ReadTableDef.h | 17 -
core/sql/sqlcat/TrafDDLdesc.cpp | 617 ++++++
core/sql/sqlcat/TrafDDLdesc.h | 1120 ++++++++++
core/sql/sqlcat/desc.h | 473 +---
core/sql/sqlcat/readRealArk.cpp | 146 +-
core/sql/sqlcat/readRealArk.h | 1 -
core/sql/sqlcomp/CmpDDLCatErrorCodes.h | 1 +
core/sql/sqlcomp/CmpDescribe.cpp | 23 +-
core/sql/sqlcomp/CmpSeabaseDDL.h | 85 +-
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 323 ++-
core/sql/sqlcomp/CmpSeabaseDDLincludes.h | 2 +-
core/sql/sqlcomp/CmpSeabaseDDLindex.cpp | 40 +-
core/sql/sqlcomp/CmpSeabaseDDLmd.h | 7 +-
core/sql/sqlcomp/CmpSeabaseDDLroutine.cpp | 1 -
core/sql/sqlcomp/CmpSeabaseDDLschema.cpp | 246 +++
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 731 +++++--
core/sql/sqlcomp/CmpSeabaseDDLview.cpp | 12 +-
core/sql/sqlcomp/DefaultConstants.h | 6 +
core/sql/sqlcomp/nadefaults.cpp | 10 +-
docs/src/site/markdown/index.md | 14 +-
.../site/resources/images/carousel/meetup.png | Bin 0 -> 81640 bytes
79 files changed, 5930 insertions(+), 3366 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/generator/GenRelScan.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/regress/seabase/EXPECTED031
----------------------------------------------------------------------
diff --cc core/sql/regress/seabase/EXPECTED031
index c3141d2,b6f3a9d..9c2255d
--- a/core/sql/regress/seabase/EXPECTED031
+++ b/core/sql/regress/seabase/EXPECTED031
@@@ -560,8 -482,8 +560,9 @@@ LC RC OP OPERATOR OP
>>invoke t031t1;
-- Definition of Trafodion table TRAFODION.SCH.T031T1
--- Definition current Tue Aug 2 19:11:21 2016
+-- Definition current Wed Aug 3 15:45:17 2016
+
+
(
SYSKEY LARGEINT NO DEFAULT NOT NULL NOT DROPPABLE
, A INT DEFAULT NULL
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/regress/seabase/TEST031
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/66d03a83/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
[5/5] incubator-trafodion git commit: Merge [TRAFODION-2138] PR-637
Hive scan on wide tables can result in lost rows or error
Posted by su...@apache.org.
Merge [TRAFODION-2138] PR-637 Hive scan on wide tables can result in lost rows or error
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/a72306e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/a72306e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/a72306e4
Branch: refs/heads/master
Commit: a72306e4edefbbb575cdc4955d4b1a40a60b0fa7
Parents: 9f3542d f5a716d
Author: Suresh Subbiah <su...@apache.org>
Authored: Sun Aug 7 23:47:33 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Sun Aug 7 23:47:33 2016 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 1 +
core/sql/executor/ExHdfsScan.cpp | 52 +++++++++----
core/sql/exp/ExpLOBaccess.cpp | 2 +-
core/sql/generator/GenRelScan.cpp | 13 +++-
core/sql/optimizer/BindRelExpr.cpp | 18 +++++
core/sql/regress/hive/EXPECTED002 | 4 +-
core/sql/regress/seabase/EXPECTED031 | 124 ++++++++++++++++++++++++------
core/sql/regress/seabase/TEST031 | 19 +++++
core/sql/sqlcomp/nadefaults.cpp | 3 +-
9 files changed, 192 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a72306e4/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/a72306e4/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
[3/5] incubator-trafodion git commit: Address comments by Hans
Posted by su...@apache.org.
Address comments by Hans
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/bfd3cdb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/bfd3cdb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/bfd3cdb0
Branch: refs/heads/master
Commit: bfd3cdb02b8057aede24c8f91d43e035e3bd0eed
Parents: 66d03a8
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Aug 3 22:30:51 2016 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Aug 3 22:30:51 2016 +0000
----------------------------------------------------------------------
core/sql/bin/SqlciErrors.txt | 2 +-
core/sql/generator/GenRelScan.cpp | 9 ++++++---
core/sql/regress/hive/EXPECTED002 | 4 ++--
3 files changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/bfd3cdb0/core/sql/bin/SqlciErrors.txt
----------------------------------------------------------------------
diff --git a/core/sql/bin/SqlciErrors.txt b/core/sql/bin/SqlciErrors.txt
index 10d0a2c..a9f08c2 100644
--- a/core/sql/bin/SqlciErrors.txt
+++ b/core/sql/bin/SqlciErrors.txt
@@ -1282,7 +1282,7 @@ $1~String1 --------------------------------
4223 42000 99999 BEGINNER MAJOR DBADMIN $0~String0 not supported in this software version.
4224 42000 99999 BEGINNER MAJOR DBADMIN The directory $0~String0 is not a valid Hive location.
4225 42000 99999 BEGINNER MAJOR DBADMIN Number of column families cannot exceed 32.
-4226 42000 99999 BEGINNER MAJOR DBADMIN Table $0~TableName has a maximum record length of $0~Int0 which is greater than HDFS_IO_BUFFERSIZE. Increase buffer size setting, or reduce HIVE_MAX_STRING_LENGTH.
+4226 42000 99999 BEGINNER MAJOR DBADMIN Table $0~TableName has a maximum record length of $0~Int0 which is greater than the HDFS_IO_BUFFERSIZE default. Increase buffer size setting, or reduce the HIVE_MAX_STRING_LENGTH default.
4240 ZZZZZ 99999 BEGINNER MAJOR DBADMIN Expression $0~string0 in the DIVISION BY clause references columns other than clustering key columns.
4241 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The value $0~string0 is not supported at this place in the DIVISION BY clause, only constants are allowed.
4242 ZZZZZ 99999 BEGINNER MAJOR DBADMIN The value $0~string0 is not supported at this place in the DIVISION BY clause, only key columns are allowed.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/bfd3cdb0/core/sql/generator/GenRelScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelScan.cpp b/core/sql/generator/GenRelScan.cpp
index b60f756..4cb64fd 100644
--- a/core/sql/generator/GenRelScan.cpp
+++ b/core/sql/generator/GenRelScan.cpp
@@ -1186,10 +1186,13 @@ if (hTabStats->isOrcFile())
{
rangeTailIOSize = getTableDesc()->getNATable()->getRecordLength() +
(getTableDesc()->getNATable()->getClusteringIndex()->
- getAllColumns().entries())*2;
+ getAllColumns().entries())*2 + 16*1024;
// for each range we look ahead in the next range upto the maximum
- // record length to find the end of record delimiter.
- rangeTailIOSize = MAXOF(rangeTailIOSize, 16*1024);
+ // record length to find the end of record delimiter. The 16KB is
+ // old default setting which worked fine till we started testing
+ // wide columns. We need to keep the 16 KB as additional fudge factor
+ // as recordlength in compiler is different from what it would be
+ // in a Hive text file
}
char * tablename =
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/bfd3cdb0/core/sql/regress/hive/EXPECTED002
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED002 b/core/sql/regress/hive/EXPECTED002
index 907d2f2..d779012 100644
--- a/core/sql/regress/hive/EXPECTED002
+++ b/core/sql/regress/hive/EXPECTED002
@@ -18,7 +18,7 @@
>>--------------------------------------------------------------------------
>>
>>log;
-hdfsBufSize_ = 3072, rangeTailIOSize_ = 16384, hdfsSqlMaxRecLen_ = 8
+hdfsBufSize_ = 3072, rangeTailIOSize_ = 16636, hdfsSqlMaxRecLen_ = 8
>>select count(*) from customer ;
(EXPR)
@@ -141,7 +141,7 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>--------------------------------------------------------------------------
>>
>>log;
-hdfsBufSize_ = 32768, rangeTailIOSize_ = 16384, hdfsSqlMaxRecLen_ = 8
+hdfsBufSize_ = 32768, rangeTailIOSize_ = 16636, hdfsSqlMaxRecLen_ = 8
>>select count(*) from customer ;
(EXPR)