You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by hz...@apache.org on 2016/05/19 21:32:47 UTC
[1/2] incubator-trafodion git commit: TRAFODION-2002 check columns
for Hive inserts
Repository: incubator-trafodion
Updated Branches:
refs/heads/master c41b39bab -> 444cd9cc8
TRAFODION-2002 check columns for Hive inserts
Also TRAFODION-1904 column list in insert/select into Hive table is ignored
Added checks for matching number and data types, so that the
data to be inserted matches the column layout of the Hive target
table.
Using a column list for insert into a Hive table now will raise
SQL error 4223 instead of being ignored (so it is still not supported).
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/3205c6a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/3205c6a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/3205c6a2
Branch: refs/heads/master
Commit: 3205c6a2470146297ea5215a3e82eab6b74c1541
Parents: 855759f
Author: Hans Zeller <hz...@apache.org>
Authored: Wed May 18 22:25:21 2016 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Wed May 18 22:25:21 2016 +0000
----------------------------------------------------------------------
core/sql/optimizer/BindRelExpr.cpp | 44 +++++++++++++++++++++++++++-
core/sql/optimizer/RelFastTransport.cpp | 4 +--
core/sql/optimizer/RelFastTransport.h | 14 ++++-----
core/sql/regress/hive/EXPECTED003 | 26 ++++++++++++++++
core/sql/regress/hive/TEST003 | 11 +++++++
5 files changed, 89 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index 0fa2288..59d0b9b 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -9150,6 +9150,14 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
bindWA->setErrStatus();
return this;
}
+
+ // specifying a list of column names to insert to is not yet supported
+ if (insertColTree_) {
+ *CmpCommon::diags() << DgSqlCode(-4223)
+ << DgString0("Target column list for insert into Hive table");
+ bindWA->setErrStatus();
+ return this;
+ }
// NABoolean isSequenceFile = (*hTabStats)[0]->isSequenceFile();
const NABoolean isSequenceFile = hTabStats->isSequenceFile();
@@ -9160,7 +9168,7 @@ RelExpr *Insert::bindNode(BindWA *bindWA)
new (bindWA->wHeap()) NAString(hiveTablePath),
new (bindWA->wHeap()) NAString(hostName),
hdfsPort,
- TRUE,
+ getTableDesc(),
new (bindWA->wHeap()) NAString(getTableName().getQualifiedNameObj().getObjectName()),
FastExtract::FILE,
bindWA->wHeap());
@@ -16676,6 +16684,40 @@ RelExpr * FastExtract::bindNode(BindWA *bindWA)
// can also get from child Root compExpr
ValueIdList vidList;
childRETDesc->getValueIdList(vidList, USER_COLUMN);
+
+ if (isHiveInsert())
+ {
+ // validate number of columns and column types of the select list
+ ValueIdList tgtCols;
+
+ hiveTableDesc_->getUserColumnList(tgtCols);
+
+ if (vidList.entries() != tgtCols.entries())
+ {
+ // 4023 degree of row value constructor must equal that of target table
+ *CmpCommon::diags() << DgSqlCode(-4023)
+ << DgInt0(vidList.entries())
+ << DgInt1(tgtCols.entries());
+ bindWA->setErrStatus();
+ return NULL;
+ }
+
+ // Check that the source and target types are compatible.
+ for (CollIndex j=0; j<vidList.entries(); j++)
+ {
+ Assign *tmpAssign = new(bindWA->wHeap())
+ Assign(tgtCols[j].getItemExpr(), vidList[j].getItemExpr());
+
+ if ( CmpCommon::getDefault(ALLOW_IMPLICIT_CHAR_CASTING) == DF_ON )
+ tmpAssign->tryToDoImplicitCasting(bindWA);
+ const NAType *targetType = tmpAssign->synthesizeType();
+ if (!targetType) {
+ bindWA->setErrStatus();
+ return NULL;
+ }
+ }
+ }
+
setSelectList(vidList);
if (includeHeader())
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/RelFastTransport.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.cpp b/core/sql/optimizer/RelFastTransport.cpp
index 5049d76..8daa6c2 100644
--- a/core/sql/optimizer/RelFastTransport.cpp
+++ b/core/sql/optimizer/RelFastTransport.cpp
@@ -49,7 +49,7 @@ FastExtract::FastExtract(const FastExtract & other)
targetName_ = other.targetName_;
hdfsHostName_ = other.hdfsHostName_;
hdfsPort_ = other.hdfsPort_;
- isHiveInsert_ = other.isHiveInsert_;
+ hiveTableDesc_ = other.hiveTableDesc_;
hiveTableName_ = other.hiveTableName_;
delimiter_ = other.delimiter_;
isAppend_ = other.isAppend_;
@@ -87,7 +87,7 @@ RelExpr * FastExtract::copyTopNode(RelExpr *derivedNode,
result->targetName_ = targetName_;
result->hdfsHostName_ = hdfsHostName_;
result->hdfsPort_ = hdfsPort_;
- result->isHiveInsert_= isHiveInsert_;
+ result->hiveTableDesc_= hiveTableDesc_;
result->hiveTableName_ = hiveTableName_;
result->delimiter_ = delimiter_;
result->isAppend_ = isAppend_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/optimizer/RelFastTransport.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelFastTransport.h b/core/sql/optimizer/RelFastTransport.h
index d576f6e..f0088e1 100644
--- a/core/sql/optimizer/RelFastTransport.h
+++ b/core/sql/optimizer/RelFastTransport.h
@@ -105,7 +105,7 @@ public :
targetName_(*targName, oHeap),
hdfsHostName_(oHeap),
hdfsPort_(0),
- isHiveInsert_(FALSE),
+ hiveTableDesc_(NULL),
delimiter_(*delim, oHeap),
isAppend_(isAppend),
includeHeader_(needsHeader),
@@ -126,7 +126,7 @@ public :
targetName_(*targName, oHeap),
hdfsHostName_(oHeap),
hdfsPort_(0),
- isHiveInsert_(FALSE),
+ hiveTableDesc_(NULL),
delimiter_(oHeap),
isAppend_(FALSE),
includeHeader_(FALSE),
@@ -146,7 +146,7 @@ public :
targetName_(oHeap),
hdfsHostName_(oHeap),
hdfsPort_(0),
- isHiveInsert_(FALSE),
+ hiveTableDesc_(NULL),
delimiter_(oHeap),
isAppend_(FALSE),
includeHeader_(FALSE),
@@ -162,7 +162,7 @@ public :
NAString* targName,
NAString* hostName,
Int32 portNum,
- NABoolean isHiveInsert,
+ TableDesc *hiveTableDesc,
NAString* hiveTableName,
ExtractDest targType,
CollHeap *oHeap = CmpCommon::statementHeap())
@@ -171,7 +171,7 @@ public :
targetName_(*targName, oHeap),
hdfsHostName_(*hostName, oHeap),
hdfsPort_(portNum),
- isHiveInsert_(isHiveInsert),
+ hiveTableDesc_(hiveTableDesc),
hiveTableName_(*hiveTableName, oHeap),
delimiter_(oHeap),
isAppend_(FALSE),
@@ -269,7 +269,7 @@ public :
const NAString& getHdfsHostName() const {return hdfsHostName_;}
Int32 getHdfsPort() const {return hdfsPort_;}
const NAString& getHiveTableName() const {return hiveTableName_;}
- NABoolean isHiveInsert() const {return isHiveInsert_;}
+ NABoolean isHiveInsert() const {return (hiveTableDesc_ != NULL);}
const NAString& getDelimiter() const {return delimiter_;}
NABoolean isAppend() const {return isAppend_;}
NABoolean includeHeader() const {return includeHeader_ ;}
@@ -321,7 +321,7 @@ private:
NAString nullString_;
NAString recordSeparator_;
NABoolean isAppend_;
- NABoolean isHiveInsert_;
+ TableDesc *hiveTableDesc_;
NAString hiveTableName_;
NABoolean overwriteHiveTable_;
NABoolean isSequenceFile_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/regress/hive/EXPECTED003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED003 b/core/sql/regress/hive/EXPECTED003
index 8c8006a..5648797 100644
--- a/core/sql/regress/hive/EXPECTED003
+++ b/core/sql/regress/hive/EXPECTED003
@@ -123,6 +123,32 @@ P_PROMO_SK P_PROMO_ID P_START_DATE_SK P_END_DATE_SK P_ITEM_S
--- 20 row(s) selected.
>>
+>>-- some negative tests
+>>insert into hive.ins_promotion (p_promo_sk, p_item_sk) select * from hive.promotion;
+
+*** ERROR[4223] Target column list for insert into Hive table not supported in this software version.
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- target column list is not supported
+>>
+>>insert into hive.ins_promotion select *, p_promo_sk from hive.promotion;
+
+*** ERROR[4023] The degree of each row value constructor (20) must equal the degree of the target table column list (19).
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- number of columns doesn't match
+>>
+>>prepare s from
++>insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f');
+
+*** ERROR[4039] Column T_TIME_SK is of type INTEGER, incompatible with the value's type, CHAR(1).
+
+*** ERROR[8822] The statement was not prepared.
+
+>>-- wrong data types
+>>
>>
>>--try new HIVE SYNTAX
>>--------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/3205c6a2/core/sql/regress/hive/TEST003
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST003 b/core/sql/regress/hive/TEST003
index f442420..ada9c87 100644
--- a/core/sql/regress/hive/TEST003
+++ b/core/sql/regress/hive/TEST003
@@ -87,6 +87,17 @@ select count(*) from hive.ins_promotion;
--select from hive table where row delimiter was explicettely specified
select [first 20] * from hive.ins_promotion order by p_promo_sk ;
+-- some negative tests
+insert into hive.ins_promotion (p_promo_sk, p_item_sk) select * from hive.promotion;
+-- target column list is not supported
+
+insert into hive.ins_promotion select *, p_promo_sk from hive.promotion;
+-- number of columns doesn't match
+
+prepare s from
+insert into hive.ins_time_dim values ('a', 2, 3, 4, 5, 6, 'c', 'd', 'e', 'f');
+-- wrong data types
+
--try new HIVE SYNTAX
--------------
[2/2] incubator-trafodion git commit: TRAFODION-2002 check columns
for Hive inserts
Posted by hz...@apache.org.
TRAFODION-2002 check columns for Hive inserts
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/444cd9cc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/444cd9cc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/444cd9cc
Branch: refs/heads/master
Commit: 444cd9cc865beb1eb2d3c7e20d465db841217c6a
Parents: c41b39b 3205c6a
Author: Hans Zeller <hz...@apache.org>
Authored: Thu May 19 21:30:19 2016 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Thu May 19 21:30:19 2016 +0000
----------------------------------------------------------------------
core/sql/optimizer/BindRelExpr.cpp | 44 +++++++++++++++++++++++++++-
core/sql/optimizer/RelFastTransport.cpp | 4 +--
core/sql/optimizer/RelFastTransport.h | 14 ++++-----
core/sql/regress/hive/EXPECTED003 | 26 ++++++++++++++++
core/sql/regress/hive/TEST003 | 11 +++++++
5 files changed, 89 insertions(+), 10 deletions(-)
----------------------------------------------------------------------