You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by hz...@apache.org on 2016/07/27 20:29:35 UTC
[1/4] incubator-trafodion git commit: [TRAFODION-2121] add support of
HIVE data type of DECIMAL
Repository: incubator-trafodion
Updated Branches:
refs/heads/master 657510219 -> f9eaa7675
[TRAFODION-2121] add support of HIVE data type of DECIMAL
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/ad6c2f3e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/ad6c2f3e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/ad6c2f3e
Branch: refs/heads/master
Commit: ad6c2f3eecde12fa50c95a1b8f22b53fb4534625
Parents: 81a455e
Author: Liu Ming <ov...@sina.com>
Authored: Sat Jul 23 23:58:56 2016 +0000
Committer: Liu Ming <ov...@sina.com>
Committed: Sat Jul 23 23:58:56 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtilGet.cpp | 2 +-
core/sql/executor/hiveHook.cpp | 4 +--
core/sql/optimizer/NATable.cpp | 52 +++++++++++++++++++++++++--
core/sql/regress/hive/EXPECTED005 | 12 +++----
core/sql/regress/hive/TEST005_a.hive.sql | 8 +++--
core/sql/regress/hive/tbl_type.data | 2 +-
6 files changed, 66 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/executor/ExExeUtilGet.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilGet.cpp b/core/sql/executor/ExExeUtilGet.cpp
index 645d381..90a8d06 100644
--- a/core/sql/executor/ExExeUtilGet.cpp
+++ b/core/sql/executor/ExExeUtilGet.cpp
@@ -5253,7 +5253,7 @@ short ExExeUtilHiveMDaccessTcb::work()
infoCol->colScale = 0;
str_pad(infoCol->dtQualifier, 28, ' ');
infoCol->dtStartField = 1;
- infoCol->dtEndField = 6;
+ infoCol->dtEndField = 3;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/executor/hiveHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/hiveHook.cpp b/core/sql/executor/hiveHook.cpp
index 7e05763..1fc032a 100644
--- a/core/sql/executor/hiveHook.cpp
+++ b/core/sql/executor/hiveHook.cpp
@@ -375,7 +375,7 @@ struct hive_column_desc* populateColumns(HiveMetaData *md, Int32 cdID,
return NULL;
NAText typeStr;
- if(!extractValueStr(md, tblStr, pos, "type:", ",",
+ if(!extractValueStr(md, tblStr, pos, "type:", ", comment",
typeStr, "populateColumns::type:###"))
return NULL;
@@ -436,7 +436,7 @@ struct hive_pkey_desc* populatePartitionKey(HiveMetaData *md, Int32 tblID,
NAText nameStr = tblStr->substr(foundB, pos-foundB);
NAText typeStr;
- if(!extractValueStr(md, tblStr, pos, "type:", ",",
+ if(!extractValueStr(md, tblStr, pos, "type:", ", comment",
typeStr, "populatePartitionKeys::type:###"))
return NULL;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index afdfaa0..ad6bf86 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -3631,9 +3631,9 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
memset(maxLen, 0, 32);
int i=0,j=0;
int copyit = 0;
-
+ int lenStr = strlen(hiveType);
//get length
- for(i = 0; i < strlen(hiveType) ; i++)
+ for(i = 0; i < lenStr ; i++)
{
if(hiveType[i] == '(') //start
{
@@ -3676,6 +3676,54 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
CharInfo::IMPLICIT);
}
+ if ( !strncmp(hiveType, "decimal", 7) )
+ {
+ Int16 i=0, pstart=0, pend=0, sstart=0, send=0, p=0, s = 0;
+ Int16 hiveTypeLen = strlen(hiveType);
+ char pstr[16], sstr[16];
+ memset(pstr,0,sizeof(pstr));
+ memset(sstr,0,sizeof(sstr));
+
+ for( i = 0; i < hiveTypeLen; i++ )
+ {
+ if(hiveType[i] == '(' )
+ {
+ pstart = i+1;
+ }
+ else if(hiveType[i] == ',')
+ {
+ pend = i;
+ sstart = i+1;
+ }
+ else if(hiveType[i] == ')')
+ {
+ send = i;
+ }
+ else
+ continue;
+ }
+ if(pend - pstart > 0)
+ {
+ strncpy(pstr,hiveType+pstart, pend-pstart);
+ p=atoi(pstr);
+ }
+ if(send - sstart > 0)
+ {
+ strncpy(sstr,hiveType+sstart,send-sstart);
+ s=atoi(sstr);
+ }
+
+ if( (p>0) && (s>0) )
+ {
+ return new (heap) SQLDecimal( p, s, TRUE, TRUE);
+ }
+ else
+ {
+ return new (heap) SQLDecimal( 10, 0, TRUE, TRUE);
+ }
+
+ }
+
return NULL;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index 75a64aa..38c91d0 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -434,10 +434,10 @@ ID CHAPTER ENGLISH TRANSLATOR
>>
>>select * from tbl_type;
-TINT SM I BIG STR F D T DT VC
----- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ----------------------------------------
+TINT SM I BIG STR F D T DT VC D10 D18
+---- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- ----------- ----------
- 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar
+ 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11
--- 1 row(s) selected.
>>insert into tbl_type_temp select * from tbl_type;
@@ -445,10 +445,10 @@ TINT SM I BIG STR F
--- 1 row(s) inserted.
>>select * from tbl_type_temp;
-TINT SM I BIG STR F D T DT VC
----- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ----------------------------------------
+TINT SM I BIG STR F D T DT VC D10 D18
+---- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- ----------- ----------
- 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar
+ 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11
--- 1 row(s) selected.
>>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/regress/hive/TEST005_a.hive.sql
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005_a.hive.sql b/core/sql/regress/hive/TEST005_a.hive.sql
index b151a79..10949dc 100644
--- a/core/sql/regress/hive/TEST005_a.hive.sql
+++ b/core/sql/regress/hive/TEST005_a.hive.sql
@@ -138,7 +138,9 @@ create external table tbl_type
d double,
t timestamp,
dt date,
- vc varchar(10)
+ vc varchar(10),
+ d10 decimal,
+ d18 decimal(8,2)
)
row format delimited fields terminated by '|'
location '/user/hive/exttables/tbl_type';
@@ -155,7 +157,9 @@ create table tbl_type_temp
d double,
t timestamp,
dt date,
- vc varchar(10)
+ vc varchar(10),
+ d10 decimal,
+ d18 decimal(8,2)
)
row format delimited fields terminated by '|';
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/ad6c2f3e/core/sql/regress/hive/tbl_type.data
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/tbl_type.data b/core/sql/regress/hive/tbl_type.data
index 99c4827..9e56472 100644
--- a/core/sql/regress/hive/tbl_type.data
+++ b/core/sql/regress/hive/tbl_type.data
@@ -1 +1 @@
-101|202|203|204|two hundred|2E2|2E+002|2022-02-02 22:22:22.222222|2022-02-02|varchar
+101|202|203|204|two hundred|2E2|2E+002|2022-02-02 22:22:22.222222|2022-02-02|varchar|1234567890|123456.11
[4/4] incubator-trafodion git commit: Merge [TRAFODION-2121] add
support of HIVE data type of DECIMAL PR-618
Posted by hz...@apache.org.
Merge [TRAFODION-2121] add support of HIVE data type of DECIMAL PR-618
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/f9eaa767
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/f9eaa767
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/f9eaa767
Branch: refs/heads/master
Commit: f9eaa767570d69eec2d383257a5c72fb260f4f51
Parents: 6575102 effe7b8
Author: Hans Zeller <hz...@apache.org>
Authored: Wed Jul 27 20:28:55 2016 +0000
Committer: Hans Zeller <hz...@apache.org>
Committed: Wed Jul 27 20:28:55 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtilGet.cpp | 2 +-
core/sql/executor/hiveHook.cpp | 4 +-
core/sql/optimizer/NATable.cpp | 81 ++++++++++++++++++++++++++-
core/sql/regress/hive/EXPECTED005 | 12 ++--
core/sql/regress/hive/TEST005_a.hive.sql | 12 +++-
core/sql/regress/hive/tbl_type.data | 2 +-
6 files changed, 99 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[2/4] incubator-trafodion git commit: fix comments issues
Posted by hz...@apache.org.
fix comments issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/d2825057
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/d2825057
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/d2825057
Branch: refs/heads/master
Commit: d2825057c8fc59f706d8d1fd26d9e266645b2f9a
Parents: ad6c2f3
Author: Liu Ming <ov...@sina.com>
Authored: Tue Jul 26 13:17:50 2016 +0000
Committer: Liu Ming <ov...@sina.com>
Committed: Tue Jul 26 13:17:50 2016 +0000
----------------------------------------------------------------------
core/sql/optimizer/NATable.cpp | 38 ++++++++++++++++++++++-----
core/sql/regress/hive/EXPECTED005 | 8 +++---
core/sql/regress/hive/TEST005_a.hive.sql | 8 ++++--
core/sql/regress/hive/tbl_type.data | 2 +-
4 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d2825057/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index ad6bf86..b59abd3 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -86,6 +86,10 @@
#include "CmpMain.h"
#define MAX_NODE_NAME 9
+#define MAX_PRECISION_ALLOWED 18
+#define HIVE_MAX_PRECISION_ALLOWED 38
+#define MAX_SCALE_ALLOWED 6
+#define MAX_NUM_LEN 16
#include "SqlParserGlobals.h"
@@ -3678,9 +3682,9 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
if ( !strncmp(hiveType, "decimal", 7) )
{
- Int16 i=0, pstart=0, pend=0, sstart=0, send=0, p=0, s = 0;
- Int16 hiveTypeLen = strlen(hiveType);
- char pstr[16], sstr[16];
+ Lng32 i=0, pstart=0, pend=0, sstart=0, send=0, p=-1, s = -1;
+ Lng32 hiveTypeLen = strlen(hiveType);
+ char pstr[MAX_NUM_LEN], sstr[MAX_NUM_LEN];
memset(pstr,0,sizeof(pstr));
memset(sstr,0,sizeof(sstr));
@@ -3702,25 +3706,47 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
else
continue;
}
+
if(pend - pstart > 0)
{
+ if( (pend - pstart) >= MAX_NUM_LEN ) // too long
+ return NULL;
strncpy(pstr,hiveType+pstart, pend-pstart);
p=atoi(pstr);
}
+
if(send - sstart > 0)
{
+ if( (send - sstart) >= MAX_NUM_LEN ) // too long
+ return NULL;
strncpy(sstr,hiveType+sstart,send-sstart);
s=atoi(sstr);
}
- if( (p>0) && (s>0) )
+ if( (p>0) && (p <= MAX_PRECISION_ALLOWED) ) //have precision between 1 - 18
{
- return new (heap) SQLDecimal( p, s, TRUE, TRUE);
+ if( ( s >=0 ) && ( s<= MAX_SCALE_ALLOWED) ) //have valid scale
+ return new (heap) SQLDecimal( p, s, TRUE, TRUE);
+ else
+ return NULL;
}
- else
+ else if( (p > MAX_PRECISION_ALLOWED) && ( p <= HIVE_MAX_PRECISION_ALLOWED) )
{
+ if ( (s>=0) && ( s< p ) ) //have valid scale
+ return new (heap) SQLBigNum( p, s, TRUE, TRUE, TRUE, NULL);
+ else
+ return NULL;
+ }
+ //no p and s given, p and s are all initial value
+ else if( ( p == -1 ) && ( s == -1 ) )
+ {
+ // hive define decimal as decimal ( 10, 0 )
return new (heap) SQLDecimal( 10, 0, TRUE, TRUE);
}
+ else
+ {
+ return NULL;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d2825057/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED005 b/core/sql/regress/hive/EXPECTED005
index 38c91d0..273184d 100644
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@ -434,10 +434,10 @@ ID CHAPTER ENGLISH TRANSLATOR
>>
>>select * from tbl_type;
-TINT SM I BIG STR F D T DT VC D10 D18
+TINT SM I BIG STR F D T DT VC D10 D18 D181 D30
---- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- ----------- ----------
- 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11
+ 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11 12345 11111111111111111111111111111
--- 1 row(s) selected.
>>insert into tbl_type_temp select * from tbl_type;
@@ -445,10 +445,10 @@ TINT SM I BIG STR F
--- 1 row(s) inserted.
>>select * from tbl_type_temp;
-TINT SM I BIG STR F D T DT VC D10 D18
+TINT SM I BIG STR F D T DT VC D10 D18 D181 D30
---- ------ ----------- -------------------- ------------------------- --------------- ------------------------- -------------------------- ---------- ---------------------------------------- ----------- ----------
- 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11
+ 101 202 203 204 two hundred 2.0000000E+002 2.00000000000000000E+002 2022-02-02 22:22:22.222222 2022-02-02 varchar 1234567890 123456.11 12345 11111111111111111111111111111
--- 1 row(s) selected.
>>
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d2825057/core/sql/regress/hive/TEST005_a.hive.sql
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/TEST005_a.hive.sql b/core/sql/regress/hive/TEST005_a.hive.sql
index 10949dc..0437bbb 100644
--- a/core/sql/regress/hive/TEST005_a.hive.sql
+++ b/core/sql/regress/hive/TEST005_a.hive.sql
@@ -140,7 +140,9 @@ create external table tbl_type
dt date,
vc varchar(10),
d10 decimal,
- d18 decimal(8,2)
+ d18 decimal(8,2),
+ d181 decimal(10),
+ d30 decimal(30)
)
row format delimited fields terminated by '|'
location '/user/hive/exttables/tbl_type';
@@ -159,7 +161,9 @@ create table tbl_type_temp
dt date,
vc varchar(10),
d10 decimal,
- d18 decimal(8,2)
+ d18 decimal(8,2),
+ d181 decimal(10),
+ d30 decimal(30)
)
row format delimited fields terminated by '|';
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/d2825057/core/sql/regress/hive/tbl_type.data
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/tbl_type.data b/core/sql/regress/hive/tbl_type.data
index 9e56472..aeb9eb4 100644
--- a/core/sql/regress/hive/tbl_type.data
+++ b/core/sql/regress/hive/tbl_type.data
@@ -1 +1 @@
-101|202|203|204|two hundred|2E2|2E+002|2022-02-02 22:22:22.222222|2022-02-02|varchar|1234567890|123456.11
+101|202|203|204|two hundred|2E2|2E+002|2022-02-02 22:22:22.222222|2022-02-02|varchar|1234567890|123456.11|12345|11111111111111111111111111111
[3/4] incubator-trafodion git commit: add more checking logic,
espcially for buffer overrun
Posted by hz...@apache.org.
add more checking logic, espcially for buffer overrun
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/effe7b84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/effe7b84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/effe7b84
Branch: refs/heads/master
Commit: effe7b849c432cba9447c75ff89285e433832c60
Parents: d282505
Author: Liu Ming <ov...@sina.com>
Authored: Wed Jul 27 14:42:25 2016 +0000
Committer: Liu Ming <ov...@sina.com>
Committed: Wed Jul 27 14:42:25 2016 +0000
----------------------------------------------------------------------
core/sql/optimizer/NATable.cpp | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/effe7b84/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index b59abd3..098e514 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -87,8 +87,6 @@
#define MAX_NODE_NAME 9
#define MAX_PRECISION_ALLOWED 18
-#define HIVE_MAX_PRECISION_ALLOWED 38
-#define MAX_SCALE_ALLOWED 6
#define MAX_NUM_LEN 16
#include "SqlParserGlobals.h"
@@ -3682,8 +3680,8 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
if ( !strncmp(hiveType, "decimal", 7) )
{
- Lng32 i=0, pstart=0, pend=0, sstart=0, send=0, p=-1, s = -1;
- Lng32 hiveTypeLen = strlen(hiveType);
+ Int32 i=0, pstart=-1, pend=-1, sstart=-1, send=-1, p=-1, s = -1;
+ Int32 hiveTypeLen = strlen(hiveType);
char pstr[MAX_NUM_LEN], sstr[MAX_NUM_LEN];
memset(pstr,0,sizeof(pstr));
memset(sstr,0,sizeof(sstr));
@@ -3706,7 +3704,12 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
else
continue;
}
-
+ if(pend == -1) // no comma found, so no sstart and send
+ {
+ pend = send;
+ send = -1;
+ s = 0;
+ }
if(pend - pstart > 0)
{
if( (pend - pstart) >= MAX_NUM_LEN ) // too long
@@ -3725,14 +3728,14 @@ NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
if( (p>0) && (p <= MAX_PRECISION_ALLOWED) ) //have precision between 1 - 18
{
- if( ( s >=0 ) && ( s<= MAX_SCALE_ALLOWED) ) //have valid scale
+ if( ( s >=0 ) && ( s<= p) ) //have valid scale
return new (heap) SQLDecimal( p, s, TRUE, TRUE);
else
return NULL;
}
- else if( (p > MAX_PRECISION_ALLOWED) && ( p <= HIVE_MAX_PRECISION_ALLOWED) )
+ else if( p > MAX_PRECISION_ALLOWED)
{
- if ( (s>=0) && ( s< p ) ) //have valid scale
+ if ( (s>=0) && ( s<= p ) ) //have valid scale
return new (heap) SQLBigNum( p, s, TRUE, TRUE, TRUE, NULL);
else
return NULL;