You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by an...@apache.org on 2016/11/22 19:43:44 UTC
[2/4] incubator-trafodion git commit: TRAFODION-2363 Add support to
return hive MD info in relational format
TRAFODION-2363 Add support to return hive MD info in relational format
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/734783c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/734783c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/734783c1
Branch: refs/heads/master
Commit: 734783c124a3ddce2352c21775de3377320bbe2d
Parents: bbbd26e
Author: Anoop Sharma <an...@esgyn.com>
Authored: Mon Nov 21 23:50:07 2016 +0000
Committer: Anoop Sharma <an...@esgyn.com>
Committed: Mon Nov 21 23:50:07 2016 +0000
----------------------------------------------------------------------
core/sql/comexe/ComTdbExeUtil.cpp | 26 +-
core/sql/comexe/ComTdbExeUtil.h | 112 ++++++--
core/sql/common/ComSmallDefs.h | 8 +
core/sql/common/NAType.cpp | 201 +++++++++++++++
core/sql/common/NAType.h | 2 +
core/sql/executor/ExExeUtil.h | 7 +-
core/sql/executor/ExExeUtilGet.cpp | 365 ++++++++++++---------------
core/sql/executor/hiveHook.cpp | 168 ++++++++++--
core/sql/generator/GenRelExeUtil.cpp | 77 ++++--
core/sql/optimizer/BindRelExpr.cpp | 8 +
core/sql/optimizer/ImplRule.cpp | 2 +-
core/sql/optimizer/NATable.cpp | 186 +-------------
core/sql/optimizer/RelRoutine.cpp | 9 +-
core/sql/optimizer/RelRoutine.h | 14 +-
core/sql/optimizer/hiveHook.h | 116 +++++----
core/sql/parser/sqlparser.y | 11 +-
core/sql/regress/seabase/EXPECTED002 | 266 ++++++++++++++-----
core/sql/regress/seabase/EXPECTED012 | 52 +++-
core/sql/regress/seabase/TEST002 | 28 ++
core/sql/regress/seabase/TEST012 | 11 +
core/sql/sqlcomp/CmpSeabaseDDLrepos.cpp | 9 +-
core/sql/ustat/hs_util.h | 3 +-
22 files changed, 1095 insertions(+), 586 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/comexe/ComTdbExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.cpp b/core/sql/comexe/ComTdbExeUtil.cpp
index d4b4c23..2e2289a 100644
--- a/core/sql/comexe/ComTdbExeUtil.cpp
+++ b/core/sql/comexe/ComTdbExeUtil.cpp
@@ -2566,7 +2566,9 @@ ComTdbExeUtilHiveMDaccess::ComTdbExeUtilHiveMDaccess(
ULng32 bufferSize,
ex_expr *scanPred,
char * hivePredStr,
- char * schemaName)
+ char * catalogName,
+ char * schemaName,
+ char * objectName)
: ComTdbExeUtil(ComTdbExeUtil::HIVE_MD_ACCESS_,
0, 0, 0, // query,querylen,querycharset
NULL, 0, // tablename,tablenamelen
@@ -2579,7 +2581,9 @@ ComTdbExeUtilHiveMDaccess::ComTdbExeUtilHiveMDaccess(
numBuffers, bufferSize),
mdType_(type),
hivePredStr_(hivePredStr),
- schema_(schemaName)
+ catalog_(catalogName),
+ schema_(schemaName),
+ object_(objectName)
{
setNodeType(ComTdb::ex_HIVE_MD_ACCESS);
}
@@ -2626,8 +2630,12 @@ Long ComTdbExeUtilHiveMDaccess::pack(void * space)
{
if (hivePredStr_)
hivePredStr_.pack(space);
+ if (catalog_)
+ catalog_.pack(space);
if (schema_)
schema_.pack(space);
+ if (object_)
+ object_.pack(space);
return ComTdbExeUtil::pack(space);
}
@@ -2638,8 +2646,12 @@ Lng32 ComTdbExeUtilHiveMDaccess::unpack(void * base, void * reallocator)
{
if (hivePredStr_.unpack(base))
return -1;
+ if (catalog_.unpack(base))
+ return -1;
if (schema_.unpack(base))
return -1;
+ if (object_.unpack(base))
+ return -1;
return ComTdbExeUtil::unpack(base, reallocator);
}
@@ -2659,11 +2671,21 @@ void ComTdbExeUtilHiveMDaccess::displayContents(Space * space,ULng32 flag)
str_sprintf(buf,"hivePredStr_ = %s", hivePredStr());
space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short));
}
+ if (getCatalog())
+ {
+ str_sprintf(buf,"catalog_ = %s", getCatalog());
+ space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short));
+ }
if (getSchema())
{
str_sprintf(buf,"schema_ = %s", getSchema());
space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short));
}
+ if (getObject())
+ {
+ str_sprintf(buf,"object_ = %s", getObject());
+ space->allocateAndCopyToAlignedSpace(buf, str_len(buf), sizeof(short));
+ }
}
if (flag & 0x00000001)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/comexe/ComTdbExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbExeUtil.h b/core/sql/comexe/ComTdbExeUtil.h
index 61df851..0ec132f 100644
--- a/core/sql/comexe/ComTdbExeUtil.h
+++ b/core/sql/comexe/ComTdbExeUtil.h
@@ -2946,9 +2946,21 @@ private:
static const ComTdbVirtTableColumnInfo hiveMDTablesColInfo[] =
{
- { "CATALOG_NAME", 0, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "SCHEMA_NAME", 1, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "TABLE_NAME", 2, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "",NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0}
+ { "CATALOG_NAME", 0, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "SCHEMA_NAME", 1, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "TABLE_NAME", 2, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "FILE_FORMAT", 3, COM_USER_COLUMN, REC_BYTE_F_ASCII, 24, FALSE , SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "CREATE_TIME", 4, COM_USER_COLUMN, REC_BIN64_SIGNED, 8, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NUM_COLS", 5, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NUM_PART_COLS", 6, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NUM_BUCKET_COLS",7, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NUM_SORT_COLS", 8, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "HIVE_OWNER", 9, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "FIELD_DELIMITER", 10, COM_USER_COLUMN, REC_BIN16_SIGNED, 2, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "RECORD_TERMINATOR", 11, COM_USER_COLUMN, REC_BIN16_SIGNED, 2, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NULL_FORMAT", 12, COM_USER_COLUMN, REC_BYTE_F_ASCII, 8, FALSE, SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "LOCATION", 13, COM_USER_COLUMN, REC_BYTE_F_ASCII, 1024, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "HIVE_TABLE_TYPE", 14, COM_USER_COLUMN, REC_BYTE_F_ASCII, 128, FALSE , SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0}
};
struct HiveMDTablesColInfoStruct
@@ -2956,27 +2968,43 @@ struct HiveMDTablesColInfoStruct
char catName[256];
char schName[256];
char tblName[256];
+ char fileFormat[24];
+ Int64 createTime;
+ Int32 numCols;
+ Int32 numPartCols;
+ Int32 numBucketCols;
+ Int32 numSortCols;
+ char hiveOwner[256];
+ Int16 fieldDelimiter;
+ Int16 recordTerminator;
+ char nullFormat[8];
+ char location[1024];
+ char hiveTableType[128];
};
static const ComTdbVirtTableColumnInfo hiveMDColumnsColInfo[] =
{
- { "CATALOG_NAME", 0, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "SCHEMA_NAME", 1, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "TABLE_NAME", 2, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "COLUMN_NAME", 3, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "SQL_DATA_TYPE", 4, COM_USER_COLUMN, REC_BYTE_F_ASCII, 24, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "FS_DATA_TYPE", 5, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "COLUMN_SIZE", 6, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "CHARACTER_SET", 7, COM_USER_COLUMN, REC_BYTE_F_ASCII, 40, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "COLUMN_PRECISION", 8, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "COLUMN_SCALE", 9, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "DT_CODE", 10, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "NULLABLE", 11, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "COLUMN_NUMBER", 12, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "DATETIME_QUALIFIER", 13, COM_USER_COLUMN, REC_BYTE_F_ASCII, 28, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "DATETIME_START_FIELD", 14, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "DATETIME_END_FIELD", 15, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
- { "DEFAULT_VALUE", 16, COM_USER_COLUMN, REC_BYTE_F_ASCII, 240, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "",NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0}
+ { "CATALOG_NAME", 0, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE, SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "SCHEMA_NAME", 1, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE, SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "TABLE_NAME", 2, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE, SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "COLUMN_NAME", 3, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE, SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "SQL_DATA_TYPE", 4, COM_USER_COLUMN, REC_BYTE_F_ASCII, 32, FALSE, SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "FS_DATA_TYPE", 5, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "HIVE_DATA_TYPE", 6, COM_USER_COLUMN, REC_BYTE_F_ASCII, 32, FALSE, SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "COLUMN_SIZE", 7, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "CHARACTER_SET", 8, COM_USER_COLUMN, REC_BYTE_F_ASCII, 40, FALSE, SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "COLUMN_PRECISION", 9, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "COLUMN_SCALE", 10, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "DT_CODE", 11, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "NULLABLE", 12, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "COLUMN_NUMBER", 13, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "PART_COL_NUMBER", 14, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "BUCKET_COL_NUMBER", 15, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "SORT_COL_NUMBER", 16, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "DATETIME_QUALIFIER", 17, COM_USER_COLUMN, REC_BYTE_F_ASCII, 28, FALSE, SQLCHARSETCODE_ISO88591, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "DATETIME_START_FIELD", 18, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "DATETIME_END_FIELD", 19, COM_USER_COLUMN, REC_BIN32_SIGNED, 4, FALSE, SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
+ { "DEFAULT_VALUE", 20, COM_USER_COLUMN, REC_BYTE_F_ASCII, 240, FALSE, SQLCHARSETCODE_UTF8, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
};
struct HiveMDColumnsColInfoStruct
@@ -2985,8 +3013,9 @@ struct HiveMDColumnsColInfoStruct
char schName[256];
char tblName[256];
char colName[256];
- char sqlDatatype[24];
+ char sqlDatatype[32];
Lng32 fsDatatype;
+ char hiveDatatype[32];
Lng32 colSize;
char charSet[40];
Lng32 colPrecision;
@@ -2994,12 +3023,23 @@ struct HiveMDColumnsColInfoStruct
Lng32 dtCode;
Lng32 nullable;
Lng32 colNum;
+ Lng32 partColNum;
+ Lng32 bucketColNum;
+ Lng32 sortColNum;
char dtQualifier[28];
Lng32 dtStartField;
Lng32 dtEndField;
char defVal[240];
};
+static const ComTdbVirtTableKeyInfo hiveMDColumnsVirtTableKeyInfo[] =
+{
+ // colname keyseqnumber tablecolnumber ordering
+ { NULL, 1, 0, 0 , 0, NULL, NULL },
+ { NULL, 2, 1, 0 , 0, NULL, NULL },
+ { NULL, 2, 2, 0 , 0, NULL, NULL }
+};
+
static const ComTdbVirtTableColumnInfo hiveMDPKeysColInfo[] =
{
{ "CATALOG_NAME", 0, COM_USER_COLUMN, REC_BYTE_F_ASCII, 256, FALSE , SQLCHARSETCODE_UNKNOWN, 0, 0, 0, 0, 0, 0, 0, COM_NO_DEFAULT, "" ,NULL,NULL,COM_UNKNOWN_DIRECTION_LIT, 0},
@@ -3137,11 +3177,15 @@ public:
ULng32 bufferSize,
ex_expr *scanExpr,
char * hivePredStr,
- char * schemaName
+ char * catalogName,
+ char * schemaName,
+ char * objectName
);
char * hivePredStr() { return hivePredStr_;}
- char * getSchema() {return schema_;}
+ char * getCatalog() { return catalog_; }
+ char * getSchema() { return schema_; }
+ char * getObject() { return object_; }
// This always returns TRUE for now
Int32 orderedQueueProtocol() const { return -1; };
@@ -3236,6 +3280,22 @@ public:
return NULL;
}
+ static Int32 getVirtTableNumKeys(char * name)
+ {
+ if (strcmp(name, "COLUMNS") == 0)
+ return sizeof(hiveMDColumnsVirtTableKeyInfo)/sizeof(ComTdbVirtTableKeyInfo);
+ else
+ return 0;
+ }
+
+ static ComTdbVirtTableKeyInfo * getVirtTableKeyInfo(char * name)
+ {
+ if (strcmp(name, "COLUMNS") == 0)
+ return (ComTdbVirtTableKeyInfo *)hiveMDColumnsVirtTableKeyInfo;
+ else
+ return NULL;
+ }
+
private:
Int32 mdType_; // 00 - 03
@@ -3244,7 +3304,11 @@ private:
NABasicPtr hivePredStr_; // 08 - 15
- NABasicPtr schema_; // 16 - 23
+ NABasicPtr catalog_; // 16 - 23
+
+ NABasicPtr schema_; // 24 - 31
+
+ NABasicPtr object_; // 32 - 39
};
class ComTdbExeUtilHBaseBulkLoad : public ComTdbExeUtil
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/common/ComSmallDefs.h
----------------------------------------------------------------------
diff --git a/core/sql/common/ComSmallDefs.h b/core/sql/common/ComSmallDefs.h
index 10a37c3..67cabfb 100644
--- a/core/sql/common/ComSmallDefs.h
+++ b/core/sql/common/ComSmallDefs.h
@@ -104,6 +104,11 @@ typedef NABoolean ComBoolean;
#define HIVE_SYSTEM_CATALOG "HIVE"
#define HIVE_SYSTEM_SCHEMA "HIVE"
+#define HIVE_SYSTEM_SCHEMA_LC "hive"
+
+// default schema name to be passed to hive methods at runtime
+#define HIVE_DEFAULT_SCHEMA_EXE "default"
+
#define HIVE_STATS_CATALOG "TRAFODION"
#define HIVE_STATS_SCHEMA "\"_HIVESTATS_\""
#define HIVE_STATS_SCHEMA_NO_QUOTES "_HIVESTATS_"
@@ -437,6 +442,9 @@ enum ComTableType { COM_REGULAR_TABLE
#define EPOCH_INITIAL_VALUE 100
+// JulianTimestamp time of UNIX "epoch", 00:00:00 Jan 1, 1970
+const Int64 COM_EPOCH_TIMESTAMP=210866760000000000LL;
+
// enums used to specify the MV REWRITE PUBLISH operations
// and SYSTEM DEFAULTS propagation
enum ComPublishMVOperationType { COM_PUBLISH_MV_CREATE,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/common/NAType.cpp
----------------------------------------------------------------------
diff --git a/core/sql/common/NAType.cpp b/core/sql/common/NAType.cpp
index 385c626..8ed7129 100644
--- a/core/sql/common/NAType.cpp
+++ b/core/sql/common/NAType.cpp
@@ -39,6 +39,7 @@
#include "ComASSERT.h"
#include "NumericType.h"
#include "CharType.h"
+#include "MiscType.h"
#include "CmpCommon.h" /* want to put NAType obj's on statement heap ... */
#include "str.h"
@@ -945,3 +946,203 @@ NABoolean NAType::isSkewBusterSupportedType() const
CharInfo::CharSet NAType::getCharSet() const
{ return CharInfo::UnknownCharSet; };
+
+#define MAX_PRECISION_ALLOWED 18
+#define MAX_NUM_LEN 16
+
+NAType* NAType::getNATypeForHive(const char* hiveType, NAMemory* heap)
+{
+ if ( !strcmp(hiveType, "tinyint"))
+ {
+ if (CmpCommon::getDefault(TRAF_TINYINT_SUPPORT) == DF_OFF)
+ return new (heap) SQLSmall(TRUE /* neg */, TRUE /* allow NULL*/, heap);
+ else
+ return new (heap) SQLTiny(TRUE /* neg */, TRUE /* allow NULL*/, heap);
+ }
+
+ if ( !strcmp(hiveType, "smallint"))
+ return new (heap) SQLSmall(TRUE /* neg */, TRUE /* allow NULL*/, heap);
+
+ if ( !strcmp(hiveType, "int"))
+ return new (heap) SQLInt(TRUE /* neg */, TRUE /* allow NULL*/, heap);
+
+ if ( !strcmp(hiveType, "bigint"))
+ return new (heap) SQLLargeInt(TRUE /* neg */, TRUE /* allow NULL*/, heap);
+
+ if ( !strcmp(hiveType, "boolean"))
+ return new (heap) SQLBooleanNative(TRUE, heap);
+
+ if ( !strcmp(hiveType, "string"))
+ {
+ Int32 len = CmpCommon::getDefaultLong(HIVE_MAX_STRING_LENGTH);
+ Int32 lenInBytes = CmpCommon::getDefaultLong(HIVE_MAX_STRING_LENGTH_IN_BYTES);
+ if( lenInBytes != 32000 )
+ len = lenInBytes;
+ NAString hiveCharset = CmpCommon::getDefaultString(HIVE_DEFAULT_CHARSET);
+ // ActiveSchemaDB()->getDefaults().getValue(HIVE_DEFAULT_CHARSET);
+ hiveCharset.toUpper();
+ CharInfo::CharSet hiveCharsetEnum = CharInfo::getCharSetEnum(hiveCharset);
+ Int32 maxNumChars = 0;
+ Int32 storageLen = len;
+ SQLVarChar * nat =
+ new (heap) SQLVarChar(CharLenInfo(maxNumChars, storageLen),
+ TRUE, // allow NULL
+ FALSE, // not upshifted
+ FALSE, // not case-insensitive
+ CharInfo::getCharSetEnum(hiveCharset),
+ CharInfo::DefaultCollation,
+ CharInfo::IMPLICIT);
+ nat->setWasHiveString(TRUE);
+ return nat;
+ }
+
+ if ( !strcmp(hiveType, "float"))
+ return new (heap) SQLReal(TRUE /* allow NULL*/, heap);
+
+ if ( !strcmp(hiveType, "double"))
+ return new (heap) SQLDoublePrecision(TRUE /* allow NULL*/, heap);
+
+ if ( !strcmp(hiveType, "timestamp"))
+ return new (heap) SQLTimestamp(TRUE /* allow NULL */ , 6, heap);
+
+ if ( !strcmp(hiveType, "date"))
+ return new (heap) SQLDate(TRUE /* allow NULL */ , heap);
+
+ if ( (!strncmp(hiveType, "varchar", 7)) ||
+ (!strncmp(hiveType, "char", 4)))
+ {
+ char maxLen[32];
+ memset(maxLen, 0, 32);
+ int i=0,j=0;
+ int copyit = 0;
+ int lenStr = strlen(hiveType);
+ //get length
+ for(i = 0; i < lenStr ; i++)
+ {
+ if(hiveType[i] == '(') //start
+ {
+ copyit=1;
+ continue;
+ }
+ else if(hiveType[i] == ')') //stop
+ break;
+ if(copyit > 0)
+ {
+ maxLen[j] = hiveType[i];
+ j++;
+ }
+ }
+ Int32 len = atoi(maxLen);
+
+ if(len == 0) return NULL; //cannot parse correctly
+
+ NAString hiveCharset = CmpCommon::getDefaultString(HIVE_DEFAULT_CHARSET);
+ hiveCharset.toUpper();
+ CharInfo::CharSet hiveCharsetEnum = CharInfo::getCharSetEnum(hiveCharset);
+ Int32 maxNumChars = 0;
+ Int32 storageLen = len;
+ if (CharInfo::isVariableWidthMultiByteCharSet(hiveCharsetEnum))
+ {
+ // For Hive VARCHARs, the number specified is the max. number of characters,
+ // while we count in bytes when using HIVE_MAX_STRING_LENGTH for Hive STRING
+ // columns. Set the max character constraint and also adjust the required storage length.
+ maxNumChars = len;
+ storageLen = len * CharInfo::maxBytesPerChar(hiveCharsetEnum);
+ }
+
+ if (!strncmp(hiveType, "char", 4))
+ return new (heap) SQLChar(CharLenInfo(maxNumChars, storageLen),
+ TRUE, // allow NULL
+ FALSE, // not upshifted
+ FALSE, // not case-insensitive
+ FALSE, // not varchar
+ CharInfo::getCharSetEnum(hiveCharset),
+ CharInfo::DefaultCollation,
+ CharInfo::IMPLICIT);
+ else
+ return new (heap) SQLVarChar(CharLenInfo(maxNumChars, storageLen),
+ TRUE, // allow NULL
+ FALSE, // not upshifted
+ FALSE, // not case-insensitive
+ CharInfo::getCharSetEnum(hiveCharset),
+ CharInfo::DefaultCollation,
+ CharInfo::IMPLICIT);
+ }
+
+ if ( !strncmp(hiveType, "decimal", 7) )
+ {
+ Int32 i=0, pstart=-1, pend=-1, sstart=-1, send=-1, p=-1, s = -1;
+ Int32 hiveTypeLen = strlen(hiveType);
+ char pstr[MAX_NUM_LEN], sstr[MAX_NUM_LEN];
+ memset(pstr,0,sizeof(pstr));
+ memset(sstr,0,sizeof(sstr));
+
+ for( i = 0; i < hiveTypeLen; i++ )
+ {
+ if(hiveType[i] == '(' )
+ {
+ pstart = i+1;
+ }
+ else if(hiveType[i] == ',')
+ {
+ pend = i;
+ sstart = i+1;
+ }
+ else if(hiveType[i] == ')')
+ {
+ send = i;
+ }
+ else
+ continue;
+ }
+ if(pend == -1) // no comma found, so no sstart and send
+ {
+ pend = send;
+ send = -1;
+ s = 0;
+ }
+ if(pend - pstart > 0)
+ {
+ if( (pend - pstart) >= MAX_NUM_LEN ) // too long
+ return NULL;
+ strncpy(pstr,hiveType+pstart, pend-pstart);
+ p=atoi(pstr);
+ }
+
+ if(send - sstart > 0)
+ {
+ if( (send - sstart) >= MAX_NUM_LEN ) // too long
+ return NULL;
+ strncpy(sstr,hiveType+sstart,send-sstart);
+ s=atoi(sstr);
+ }
+
+ if( (p>0) && (p <= MAX_PRECISION_ALLOWED) ) //have precision between 1 - 18
+ {
+ if( ( s >=0 ) && ( s<= p) ) //have valid scale
+ return new (heap) SQLDecimal( p, s, TRUE, TRUE);
+ else
+ return NULL;
+ }
+ else if( p > MAX_PRECISION_ALLOWED)
+ {
+ if ( (s>=0) && ( s<= p ) ) //have valid scale
+ return new (heap) SQLBigNum( p, s, TRUE, TRUE, TRUE, NULL);
+ else
+ return NULL;
+ }
+ //no p and s given, p and s are all initial value
+ else if( ( p == -1 ) && ( s == -1 ) )
+ {
+ // hive define decimal as decimal ( 10, 0 )
+ return new (heap) SQLDecimal( 10, 0, TRUE, TRUE);
+ }
+ else
+ {
+ return NULL;
+ }
+
+ }
+
+ return NULL;
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/common/NAType.h
----------------------------------------------------------------------
diff --git a/core/sql/common/NAType.h b/core/sql/common/NAType.h
index dd81188..06755ea 100644
--- a/core/sql/common/NAType.h
+++ b/core/sql/common/NAType.h
@@ -587,6 +587,8 @@ public:
virtual NABoolean expConvSupported
(const NAType &otherNAType) const { return TRUE; }
+ static NAType* getNATypeForHive(const char* hiveType, NAMemory* heap);
+
protected:
// ---------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index f3fccb8..e3efca5 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -3245,6 +3245,7 @@ protected:
enum Step
{
INITIAL_,
+ READ_HIVE_MD_,
POSITION_,
FETCH_TABLE_,
FETCH_COLUMN_,
@@ -3262,14 +3263,16 @@ protected:
HiveMetaData* hiveMD_;
hive_column_desc * currColDesc_;
+ hive_pkey_desc * currPartnDesc_;
hive_bkey_desc * currKeyDesc_;
-
+ Int32 currColNum_;
+
char * mdRow_;
LIST (NAText *) tblNames_;
- short pos_;
char hiveCat_[1024];
char hiveSch_[1024];
+ char schForHive_[1024];
};
//////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/executor/ExExeUtilGet.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilGet.cpp b/core/sql/executor/ExExeUtilGet.cpp
index c818e14..0081cb0 100644
--- a/core/sql/executor/ExExeUtilGet.cpp
+++ b/core/sql/executor/ExExeUtilGet.cpp
@@ -66,6 +66,9 @@
#include "ExpHbaseInterface.h"
#include "sql_buffer_size.h"
#include "hdfs.h"
+
+#include "NAType.h"
+
//******************************************************************************
// *
// These definitions were stolen from CatWellKnownTables.h
@@ -361,8 +364,8 @@ static const QueryString getTrafViewsInSchemaQuery[] =
static const QueryString getTrafObjectsInViewQuery[] =
{
- {" select trim(T.schema_name) || '.' || trim(T.object_name) from "},
- {" %s.\"%s\".%s VU, %s.\"%s\".%s T "},
+ {" select trim(T.schema_name) || '.' || trim(T.object_name), trim(T.object_type) "},
+ {" from %s.\"%s\".%s VU, %s.\"%s\".%s T "},
{" where VU.using_view_uid = "},
{" (select T2.object_uid from %s.\"%s\".%s T2 "},
{" where T2.catalog_name = '%s' and "},
@@ -1754,6 +1757,8 @@ short ExExeUtilGetMetadataInfoTcb::work()
param_[9] = getMItdb().cat_;
param_[10] = getMItdb().sch_;
param_[11] = getMItdb().obj_;
+
+ numOutputEntries_ = 2;
}
break;
@@ -3605,7 +3610,8 @@ short ExExeUtilGetHiveMetadataInfoTcb::fetchAllHiveRows(Queue * &infoList,
rc = 0;
char buf[2000];
- str_sprintf(buf, "select rtrim(table_name) from table(hivemd(tables, %s.%s))", getMItdb().getCat(), getMItdb().getSch());
+ str_sprintf(buf, "select rtrim(table_name) from table(hivemd(tables, \"%s\"))",
+ getMItdb().getSch());
cliRC = fetchAllRows(infoList, buf, 1, TRUE, rc, FALSE);
return cliRC;
@@ -4850,10 +4856,8 @@ ExExeUtilHiveMDaccessTcb::ExExeUtilHiveMDaccessTcb(
hiveMD_(NULL),
currColDesc_(NULL),
currKeyDesc_(NULL),
- tblNames_(getHeap()),
- pos_(0)
+ tblNames_(getHeap())
{
- // queryBuf_ = new(glob->getDefaultHeap()) char[4096];
step_ = INITIAL_;
mdRow_ = new(getHeap()) char[exe_util_tdb.outputRowlen_];
@@ -4867,119 +4871,20 @@ ExExeUtilHiveMDaccessTcb::~ExExeUtilHiveMDaccessTcb()
// should move this method to common dir.
Lng32 ExExeUtilHiveMDaccessTcb::getFSTypeFromHiveColType(const char* hiveType)
{
- if ( !strcmp(hiveType, "tinyint"))
- return REC_BIN16_SIGNED;
-
- if ( !strcmp(hiveType, "smallint"))
- return REC_BIN16_SIGNED;
-
- if ( !strcmp(hiveType, "int"))
- return REC_BIN32_SIGNED;
-
- if ( !strcmp(hiveType, "bigint"))
- return REC_BIN64_SIGNED;
-
- if ( !strcmp(hiveType, "string"))
- return REC_BYTE_V_ASCII;
-
- if ( !strcmp(hiveType, "float"))
- return REC_FLOAT32;
-
- if ( !strcmp(hiveType, "double"))
- return REC_FLOAT64;
-
- if ( !strcmp(hiveType, "timestamp"))
- return REC_DATETIME;
-
- if ( !strcmp(hiveType, "date"))
- return REC_DATETIME;
-
- if ( !strncmp(hiveType, "varchar",7) )
- return REC_BYTE_V_ASCII;
-
- return -1;
+ Lng32 fstype = -1;
+ NAType * nat = NAType::getNATypeForHive(hiveType, getHeap());
+ fstype = nat->getFSDatatype();
+ delete nat;
+ return fstype;
}
Lng32 ExExeUtilHiveMDaccessTcb::getLengthFromHiveColType(const char* hiveType)
{
- if ( !strcmp(hiveType, "tinyint"))
- return 2;
-
- if ( !strcmp(hiveType, "smallint"))
- return 2;
-
- if ( !strcmp(hiveType, "int"))
- return 4;
-
- if ( !strcmp(hiveType, "bigint"))
- return 8;
-
- if ( !strcmp(hiveType, "string")) {
- char maxStrLen[100];
- char maxStrLenInBytes[100];
- cliInterface()->getCQDval("HIVE_MAX_STRING_LENGTH", maxStrLen);
- cliInterface()->getCQDval("HIVE_MAX_STRING_LENGTH_IN_BYTES", maxStrLenInBytes);
- //Hive varchar(n) contains n character instead of n bytes
- //so trafodion map hive varchar(n) into Trafodion varchar(n)
- //but hive string will map to Trafodion varchar(n BYTES)
- //So this CQD will be confusing
- //We change the CQD name to explicitly indicate it is lenght in bytes
- //For backward compatibility, HIVE_MAX_STRING_LENGTH still remains now, but is deprecated, user can still use it
- //But HIVE_MAX_STRING_LENGTH_IN_BYTES will overwrite HIVE_MAX_STRING_LENGTH if changed
- Int32 hiveMaxLenInBytes = atoi(maxStrLenInBytes);
- Int32 hiveMaxLen = atoi(maxStrLen);
- if( hiveMaxLenInBytes != 32000 ) //HIVE_MAX_STRING_LENGTH_IN_BYTES changed
- return hiveMaxLenInBytes;
- else
- return hiveMaxLen;
- }
-
- if ( !strcmp(hiveType, "float"))
- return 4;
-
- if ( !strcmp(hiveType, "double"))
- return 8;
-
- if ( !strcmp(hiveType, "timestamp"))
- return 26; //Is this internal or display length? REC_DATETIME;
-
- if ( !strcmp(hiveType, "date"))
- return 10; //Is this internal or display length? REC_DATETIME;
-
- if ( !strncmp(hiveType, "varchar",7) )
- {
- //try to get the length
- char maxLen[32];
- memset(maxLen, 0, 32);
- Int32 i=0,j=0;
- Int16 copyit = 0;
- Int32 hiveTypeLen = strlen(hiveType);
-
- if( hiveTypeLen > 39) return -1;
-
- for(i = 0; i < hiveTypeLen ; i++)
- {
- if(hiveType[i] == '(')
- {
- copyit=1;
- continue;
- }
- else if(hiveType[i] == ')')
- break;
- if(copyit == 1 )
- {
- maxLen[j] = hiveType[i];
- j++;
- }
- }
-
- Int32 len = atoi(maxLen);
-
- if (len == 0) return -1;
- else
- return len;
- }
- return -1;
+ Lng32 len = -1;
+ NAType * nat = NAType::getNATypeForHive(hiveType, getHeap());
+ len = nat->getNominalSize();
+ delete nat;
+ return len;
}
short ExExeUtilHiveMDaccessTcb::work()
@@ -5017,34 +4922,26 @@ short ExExeUtilHiveMDaccessTcb::work()
{
if (hiveMD_)
NADELETEBASIC(hiveMD_, getHeap());
-
- char val[5];
- cliInterface()->getCQDval("HIVE_CATALOG", hiveCat_);
- cliInterface()->getCQDval("HIVE_DEFAULT_SCHEMA", hiveSch_);
-
- char userTblSch[256];
-
- cliInterface()->getCQDval("HIVE_DEFAULT_SCHEMA", userTblSch);
- NAString hiveDefaultSch(userTblSch);
- hiveDefaultSch.toLower();
- // the current schema name has been lower cased in the tdb
hiveMD_ = new (getHeap()) HiveMetaData();
- char* currSch = hiveMDtdb().getSchema();
-// change schema name to "default", since the default schema name Hive uses,
-//if necessary. In our stack the default hive schema name is usually "HIVE"
- if (!strcmp(hiveDefaultSch.data(), currSch))
- currSch = (char *) hiveMD_->getDefaultSchemaName();
-
- NABoolean readEntireSchema = FALSE;
- if (hiveMDtdb().mdType_ != ComTdbExeUtilHiveMDaccess::TABLES_) {
- readEntireSchema = TRUE;
- }
+ if (hiveMDtdb().getCatalog())
+ strcpy(hiveCat_, hiveMDtdb().getCatalog());
+
+ if ((! hiveMDtdb().getSchema()) ||
+ (! strcmp(hiveMDtdb().getSchema(), HIVE_SYSTEM_SCHEMA_LC)) ||
+ (! strcmp(hiveMDtdb().getSchema(), HIVE_SYSTEM_SCHEMA)))
+ {
+ strcpy(hiveSch_, HIVE_SYSTEM_SCHEMA_LC);
+ strcpy(schForHive_, HIVE_DEFAULT_SCHEMA_EXE);
+ }
+ else
+ {
+ strcpy(hiveSch_, hiveMDtdb().getSchema());
+ strcpy(schForHive_, hiveSch_);
+ }
- retStatus = hiveMD_->init(readEntireSchema,
- currSch,
- hiveMDtdb().hivePredStr());
+ retStatus = hiveMD_->init();
if (!retStatus)
{
*diags << DgSqlCode(-1190)
@@ -5055,36 +4952,56 @@ short ExExeUtilHiveMDaccessTcb::work()
step_ = HANDLE_ERROR_;
break;
}
- if (!readEntireSchema)
- {
- HVC_RetCode retCode = hiveMD_->getClient()->
- getAllTables(currSch, tblNames_);
- if ((retCode != HVC_OK) && (retCode != HVC_DONE))
- {
- *diags << DgSqlCode(-1190)
- << DgString0((char*)
- "HiveClient_JNI::getAllTables()")
- << DgString1(hiveMD_->getClient()->
- getErrorText(retCode))
- << DgInt0(retCode)
- << DgString2(GetCliGlobals()->getJniErrorStr());
- step_ = HANDLE_ERROR_;
- break;
- }
- }
- step_ = POSITION_;
+
+ step_ = READ_HIVE_MD_;
}
break;
+ case READ_HIVE_MD_:
+ {
+ char* currSch = schForHive_;
+ char* currObj = hiveMDtdb().getObject();
+
+ if (! currObj)
+ {
+ HVC_RetCode retCode = hiveMD_->getClient()->
+ getAllTables(currSch, tblNames_);
+ if ((retCode != HVC_OK) && (retCode != HVC_DONE))
+ {
+ *diags << DgSqlCode(-1190)
+ << DgString0((char*)
+ "HiveClient_JNI::getAllTables()")
+ << DgString1(hiveMD_->getClient()->
+ getErrorText(retCode))
+ << DgInt0(retCode)
+ << DgString2(GetCliGlobals()->getJniErrorStr());
+ step_ = HANDLE_ERROR_;
+ break;
+ }
+ }
+ else
+ {
+ NAText * nat = new(getHeap()) NAText(currObj);
+ tblNames_.insert(nat);
+ }
+
+ // read info for entries specified in tblNames_
+ int i = 0;
+ while (i < tblNames_.entries())
+ {
+ hiveMD_->getTableDesc(schForHive_, tblNames_[i]->c_str());
+ i++;
+ }
+
+ step_ = POSITION_;
+ }
+ break;
+
case POSITION_:
{
hive_tbl_desc * htd = NULL;
- if (hiveMDtdb().mdType_ != ComTdbExeUtilHiveMDaccess::TABLES_) {
- hiveMD_->position();
- htd = hiveMD_->getNext();
- }
- else
- pos_ = 0; // we are not reading the entire schema.
+ hiveMD_->position();
+ htd = hiveMD_->getNext();
if (hiveMDtdb().mdType_ == ComTdbExeUtilHiveMDaccess::TABLES_)
{
@@ -5092,10 +5009,18 @@ short ExExeUtilHiveMDaccessTcb::work()
}
else if (hiveMDtdb().mdType_ == ComTdbExeUtilHiveMDaccess::COLUMNS_)
{
+ currColNum_ = 0;
+
if (htd)
- currColDesc_ = htd->getColumns();
+ {
+ currColDesc_ = htd->getColumns();
+ currPartnDesc_ = htd->getPartKey();
+ }
else
- currColDesc_ = NULL;
+ {
+ currColDesc_ = NULL;
+ currPartnDesc_ = NULL;
+ }
step_ = FETCH_COLUMN_;
}
@@ -5121,46 +5046,63 @@ short ExExeUtilHiveMDaccessTcb::work()
if (qparent_.up->isFull())
return WORK_OK;
- if (hiveMDtdb().mdType_ != ComTdbExeUtilHiveMDaccess::TABLES_) {
- if (hiveMD_->atEnd())
+ if (hiveMD_->atEnd())
{
step_ = DONE_;
break;
}
- }
- else {
- if (pos_ >= tblNames_.entries())
- {
- step_ = DONE_;
- break;
- }
- }
HiveMDTablesColInfoStruct *s =(HiveMDTablesColInfoStruct*)mdRow_;
str_cpy(s->catName, hiveCat_, 256, ' ');
str_cpy(s->schName, hiveSch_, 256, ' ');
- if (hiveMDtdb().mdType_ != ComTdbExeUtilHiveMDaccess::TABLES_) {
- struct hive_tbl_desc * htd = hiveMD_->getNext();
- str_cpy(s->tblName, htd->tblName_, 256, ' ');
- }
- else {
- str_cpy(s->tblName, tblNames_[pos_]->c_str(), 256, ' ');
- // delete tblNames_[pos_];
- //delete the allocation by StringArrayList::get(i)
- }
+ struct hive_tbl_desc * htd = hiveMD_->getNext();
+ str_cpy(s->tblName, htd->tblName_, 256, ' ');
+ memset(s->fileFormat, ' ', 24);
+ if (htd->getSDs())
+ {
+ if (htd->getSDs()->isOrcFile())
+ str_cpy(s->fileFormat, "ORC", 24, ' ');
+ else if (htd->getSDs()->isTextFile())
+ str_cpy(s->fileFormat, "TEXTFILE", 24, ' ');
+ else if (htd->getSDs()->isSequenceFile())
+ str_cpy(s->fileFormat, "SEQUENCE", 24, ' ');
+ }
+
+ // htd->creationTS_ is the number of seconds from epoch.
+ // convert it to juliantimestamp
+ s->createTime = htd->creationTS_*1000000 + COM_EPOCH_TIMESTAMP;
+
+ s->numCols = htd->getNumOfCols();
+ s->numPartCols = htd->getNumOfPartCols();
+ s->numSortCols = htd->getNumOfSortCols();
+ s->numBucketCols = htd->getNumOfBucketCols();
+
+ s->fieldDelimiter = htd->getSDs()->fieldTerminator_;
+ s->recordTerminator = htd->getSDs()->recordTerminator_;
+ memset(s->nullFormat, ' ', 8);
+ if (htd->getSDs()->nullFormat_)
+ str_cpy(s->nullFormat, htd->getSDs()->nullFormat_, 8, ' ');
+
+ str_cpy(s->location, htd->getSDs()->location_, 1024, ' ');
+
+ str_cpy(s->hiveTableType, htd->tableType_, 128, ' ');
+
+ str_cpy(s->hiveOwner, htd->owner_, 256, ' ');
+
step_ = APPLY_PRED_;
}
break;
- case FETCH_COLUMN_: //does not work with JNI
+ case FETCH_COLUMN_:
{
if (qparent_.up->isFull())
return WORK_OK;
- if (! currColDesc_)
+ if ((! currColDesc_) &&
+ (! currPartnDesc_))
{
step_ = DONE_;
break;
@@ -5168,6 +5110,7 @@ short ExExeUtilHiveMDaccessTcb::work()
struct hive_tbl_desc * htd = hiveMD_->getNext();
struct hive_column_desc * hcd = currColDesc_;
+ struct hive_pkey_desc * hpd = currPartnDesc_;
HiveMDColumnsColInfoStruct *infoCol =
(HiveMDColumnsColInfoStruct*)mdRow_;
@@ -5175,13 +5118,17 @@ short ExExeUtilHiveMDaccessTcb::work()
str_cpy(infoCol->catName, hiveCat_, 256, ' ');
str_cpy(infoCol->schName, hiveSch_, 256, ' ');
str_cpy(infoCol->tblName, htd->tblName_, 256, ' ');
- str_cpy(infoCol->colName, hcd->name_, 256, ' ');
+ str_cpy(infoCol->colName,
+ (hcd ? hcd->name_ : hpd->name_), 256, ' ');
+
+ infoCol->fsDatatype =
+ getFSTypeFromHiveColType(hcd ? hcd->type_ : hpd->type_);
- infoCol->fsDatatype = getFSTypeFromHiveColType(hcd->type_);
if (infoCol->fsDatatype < 0)
{
char strP[300];
- sprintf(strP, "Datatype %s is not supported.", hcd->type_);
+ sprintf(strP, "Datatype %s is not supported.",
+ (hcd ? hcd->type_ : hpd->type_));
*diags << DgSqlCode(-CLI_GET_METADATA_INFO_ERROR)
<< DgString0(strP);
@@ -5189,9 +5136,15 @@ short ExExeUtilHiveMDaccessTcb::work()
break;
}
- const char * sdtStr = Descriptor::ansiTypeStrFromFSType(infoCol->fsDatatype);
- str_cpy(infoCol->sqlDatatype, sdtStr, 24, ' ');
- infoCol->colSize = getLengthFromHiveColType(hcd->type_);
+ const char * sdtStr =
+ Descriptor::ansiTypeStrFromFSType(infoCol->fsDatatype);
+ str_cpy(infoCol->sqlDatatype, sdtStr, 32, ' ');
+
+ str_cpy(infoCol->hiveDatatype, (hcd ? hcd->type_ : hpd->type_),
+ 32, ' ');
+
+ infoCol->colSize =
+ getLengthFromHiveColType(hcd ? hcd->type_ : hpd->type_);
infoCol->colScale = 0;
// only iso charset
@@ -5203,7 +5156,7 @@ short ExExeUtilHiveMDaccessTcb::work()
infoCol->colPrecision = 0;
infoCol->nullable = 1;
- infoCol->colNum = hcd->intIndex_;
+
infoCol->dtCode = 0;
infoCol->dtStartField = 0;
infoCol->dtEndField = 0;
@@ -5231,6 +5184,13 @@ short ExExeUtilHiveMDaccessTcb::work()
// no default value
str_cpy(infoCol->defVal, " ", 240, ' ');
+ infoCol->colNum = currColNum_++;
+ infoCol->partColNum = hcd ? -1 : hpd->idx_;
+ infoCol->bucketColNum =
+ htd->getBucketColNum(hcd ? hcd->name_ : hpd->name_);
+ infoCol->sortColNum =
+ htd->getSortColNum(hcd ? hcd->name_ : hpd->name_);
+
step_ = APPLY_PRED_;
}
break;
@@ -5310,22 +5270,33 @@ short ExExeUtilHiveMDaccessTcb::work()
{
if (hiveMDtdb().mdType_ == ComTdbExeUtilHiveMDaccess::TABLES_)
{
- pos_++;
+ // move to the next table
+ hiveMD_->advance();
+
step_ = FETCH_TABLE_;
}
+
// next two else blocks do not work with JNI
else if (hiveMDtdb().mdType_ == ComTdbExeUtilHiveMDaccess::COLUMNS_)
{
if (currColDesc_)
currColDesc_ = currColDesc_->next_;
-
- if (! currColDesc_)
+ else if (currPartnDesc_)
+ currPartnDesc_ = currPartnDesc_->next_;
+
+ if ((! currColDesc_) &&
+ (! currPartnDesc_))
{
+ currColNum_ = 0;
+
// move to the next table
hiveMD_->advance();
if (! hiveMD_->atEnd())
- currColDesc_ = hiveMD_->getNext()->getColumns();
+ {
+ currColDesc_ = hiveMD_->getNext()->getColumns();
+ currPartnDesc_ = hiveMD_->getNext()->getPartKey();
+ }
}
step_ = FETCH_COLUMN_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/executor/hiveHook.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/hiveHook.cpp b/core/sql/executor/hiveHook.cpp
index 1fc032a..72bc6e4 100644
--- a/core/sql/executor/hiveHook.cpp
+++ b/core/sql/executor/hiveHook.cpp
@@ -83,9 +83,7 @@ HiveMetaData::~HiveMetaData()
}
}
-NABoolean HiveMetaData::init(NABoolean readEntireSchema,
- const char * hiveSchemaName,
- const char * tabSearchPredStr)
+NABoolean HiveMetaData::init()
{
CollHeap *h = CmpCommon::contextHeap();
@@ -93,25 +91,6 @@ NABoolean HiveMetaData::init(NABoolean readEntireSchema,
if (!connect())
return FALSE; // errCode_ should be set
- if (!readEntireSchema)
- return TRUE;
-
- int i = 0 ;
- LIST(NAText *) tblNames(h);
- HVC_RetCode retCode = client_->getAllTables(hiveSchemaName, tblNames);
- if ((retCode != HVC_OK) && (retCode != HVC_DONE)) {
- return recordError((Int32)retCode, "HiveClient_JNI::getAllTables()");
- }
-
- while (i < tblNames.entries())
- {
- getTableDesc(hiveSchemaName, tblNames[i]->c_str());
- delete tblNames[i];
- }
-
- currDesc_ = 0;
- //disconnect();
-
return TRUE;
}
@@ -283,6 +262,14 @@ struct hive_sd_desc* populateSD(HiveMetaData *md, Int32 mainSdID,
outputStr, "populateSD:outputFormat:###"))
return NULL;
+ NAText compressedStr;
+ NABoolean isCompressed = FALSE;
+ if(!extractValueStr(md, tblStr, pos, "compressed:", ",",
+ compressedStr, "populateSD:compressed:###"))
+ return NULL;
+ if (compressedStr == "true")
+ isCompressed = TRUE;
+
NAText numBucketsStr;
if(!extractValueStr(md, tblStr, pos, "numBuckets:", ",",
numBucketsStr, "populateSD:numBuckets:###"))
@@ -318,7 +305,8 @@ struct hive_sd_desc* populateSD(HiveMetaData *md, Int32 mainSdID,
newSortCols,
newBucketingCols,
fieldTerminator,
- recordTerminator
+ recordTerminator,
+ isCompressed
);
result = newSD;
@@ -680,9 +668,11 @@ struct hive_tbl_desc* HiveMetaData::getFakedTableDesc(const char* tblName)
hive_sd_desc* sd1 = new (h)hive_sd_desc(1, "loc", 0, 1, "ift", "oft", NULL,
hive_sd_desc::TABLE_SD, c1,
- sk1, bk1, '\010', '\n');
+ sk1, bk1, '\010', '\n',
+ FALSE);
- hive_tbl_desc* tbl1 = new (h) hive_tbl_desc(1, "myHive", "default",
+ hive_tbl_desc* tbl1 = new (h) hive_tbl_desc(1, "myHive", "default", "me",
+ "MANAGED",
0, sd1, 0);
return tbl1;
@@ -758,6 +748,12 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc(const char* schemaName,
schNameStr, "getTableDesc::dbName:###"))
return NULL;
+ NAText ownerStr;
+ pos = 0;
+ if(!extractValueStr(this, tblStr, pos, "owner:", ",",
+ ownerStr, "getTableDesc:owner:###"))
+ return NULL;
+
NAText createTimeStr;
pos = 0;
if(!extractValueStr(this, tblStr, pos, "createTime:", ",",
@@ -774,11 +770,19 @@ struct hive_tbl_desc* HiveMetaData::getTableDesc(const char* schemaName,
struct hive_pkey_desc* pkey = populatePartitionKey(this, 0,
tblStr, pos);
+ NAText tableTypeStr;
+ pos = 0;
+ if(!extractValueStr(this, tblStr, pos, "tableType:", ")",
+ tableTypeStr, "getTableDesc:tableType:###"))
+ return NULL;
+
result =
new (CmpCommon::contextHeap())
struct hive_tbl_desc(0, // no tblID with JNI
tblNameStr.c_str(),
schNameStr.c_str(),
+ ownerStr.c_str(),
+ tableTypeStr.c_str(),
creationTS,
sd, pkey);
@@ -816,6 +820,27 @@ NABoolean HiveMetaData::validate(Int32 tableId, Int64 redefTS,
return result;
}
+hive_tbl_desc::hive_tbl_desc(Int32 tblID, const char* name, const char* schName,
+ const char * owner,
+ const char * tableType,
+ Int64 creationTS, struct hive_sd_desc* sd,
+ struct hive_pkey_desc* pk)
+ : tblID_(tblID), sd_(sd), creationTS_(creationTS), pkey_(pk), next_(NULL)
+{
+ tblName_ = strduph(name, CmpCommon::contextHeap());
+ schName_ = strduph(schName, CmpCommon::contextHeap());
+
+ if (owner)
+ owner_ = strduph(owner, CmpCommon::contextHeap());
+ else
+ owner_ = NULL;
+
+ if (tableType)
+ tableType_ = strduph(tableType, CmpCommon::contextHeap());
+ else
+ tableType_ = NULL;
+}
+
struct hive_column_desc* hive_tbl_desc::getColumns()
{
struct hive_sd_desc* sd = sd_;
@@ -852,6 +877,18 @@ struct hive_skey_desc* hive_tbl_desc::getSortKeys()
return NULL;
}
+Int32 hive_tbl_desc::getNumOfCols()
+{
+ Int32 result = 0;
+ hive_column_desc *cd = getColumns();
+ while (cd)
+ {
+ result++;
+ cd = cd->next_;
+ }
+ return result;
+}
+
Int32 hive_tbl_desc::getNumOfPartCols() const
{
Int32 result = 0;
@@ -864,6 +901,87 @@ Int32 hive_tbl_desc::getNumOfPartCols() const
return result;
}
+Int32 hive_tbl_desc::getNumOfSortCols()
+{
+ Int32 result = 0;
+ hive_skey_desc *sk = getSortKeys();
+ while (sk)
+ {
+ result++;
+ sk = sk->next_;
+ }
+ return result;
+}
+
+Int32 hive_tbl_desc::getNumOfBucketCols()
+{
+ Int32 result = 0;
+ hive_bkey_desc *bc = getBucketingKeys();
+ while (bc)
+ {
+ result++;
+ bc = bc->next_;
+ }
+ return result;
+}
+
+Int32 hive_tbl_desc::getPartColNum(const char* name)
+{
+ Int32 num = 0;
+
+ hive_pkey_desc * desc = getPartKey();
+ while (desc)
+ {
+ if (strcmp(name, desc->name_) == 0)
+ {
+ return num;
+ }
+
+ num++;
+ desc = desc->next_;
+ }
+
+ return -1;
+}
+
+Int32 hive_tbl_desc::getBucketColNum(const char* name)
+{
+ Int32 num = 0;
+
+ hive_bkey_desc * desc = getBucketingKeys();
+ while (desc)
+ {
+ if (strcmp(name, desc->name_) == 0)
+ {
+ return num;
+ }
+
+ num++;
+ desc = desc->next_;
+ }
+
+ return -1;
+}
+
+Int32 hive_tbl_desc::getSortColNum(const char* name)
+{
+ Int32 num = 0;
+
+ hive_skey_desc * desc = getSortKeys();
+ while (desc)
+ {
+ if (strcmp(name, desc->name_) == 0)
+ {
+ return num;
+ }
+
+ num++;
+ desc = desc->next_;
+ }
+
+ return -1;
+}
+
Int64 hive_tbl_desc::redeftime()
{
Int64 result = creationTS_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/generator/GenRelExeUtil.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelExeUtil.cpp b/core/sql/generator/GenRelExeUtil.cpp
index ee089b4..9f8d2ca 100644
--- a/core/sql/generator/GenRelExeUtil.cpp
+++ b/core/sql/generator/GenRelExeUtil.cpp
@@ -4346,6 +4346,23 @@ const char * HiveMDaccessFunc::getVirtualTableName()
return "HIVEMD__";
}
+NABoolean HiveMDaccessFunc::isHiveMD(const NAString &name)
+{
+ if (memcmp(name.data(), (char*)"HIVEMD_", strlen("HIVEMD_")) == 0)
+ return TRUE;
+
+ return FALSE;
+}
+
+NAString HiveMDaccessFunc::getMDType(const NAString &name)
+{
+ NAString mdType(name);
+ mdType = mdType.remove(0, strlen("HIVEMD_"));
+ mdType = mdType.strip(NAString::trailing, '_');
+
+ return mdType;
+}
+
TrafDesc *HiveMDaccessFunc::createVirtualTableDesc()
{
TrafDesc * table_desc =
@@ -4353,7 +4370,8 @@ TrafDesc *HiveMDaccessFunc::createVirtualTableDesc()
getVirtualTableName(),
ComTdbExeUtilHiveMDaccess::getVirtTableNumCols((char*)mdType_.data()),
ComTdbExeUtilHiveMDaccess::getVirtTableColumnInfo((char*)mdType_.data()),
- 0, NULL);
+ ComTdbExeUtilHiveMDaccess::getVirtTableNumKeys((char*)mdType_.data()),
+ ComTdbExeUtilHiveMDaccess::getVirtTableKeyInfo((char*)mdType_.data()));
return table_desc;
}
@@ -4570,19 +4588,28 @@ short HiveMDaccessFunc::codeGen(Generator * generator)
}
}
+ char * catalogName = NULL;
+ NAString catalogNameInt;
+ catalogNameInt = CmpCommon::getDefaultString(HIVE_CATALOG);
+ catalogNameInt.toLower();
+ catalogName = space->allocateAlignedSpace(catalogNameInt.length() + 1);
+ strcpy(catalogName, catalogNameInt.data());
+
char * schemaName = NULL;
NAString schemaNameInt ;
- if (schemaName_.getSchemaName().isNull()) {
- const CorrName& name = getTableDesc()->getNATable()->getTableName();
- schemaNameInt = name.getQualifiedNameObj().getSchemaName();
- }
- else {
- schemaNameInt = schemaName_.getSchemaName();
- }
- schemaNameInt.toLower();
+ if (schemaName_.isNull())
+ schemaNameInt = HIVE_SYSTEM_SCHEMA_LC;
+ else
+ schemaNameInt = schemaName_;
schemaName = space->allocateAlignedSpace(schemaNameInt.length() + 1);
strcpy(schemaName, schemaNameInt.data());
+ char * objectName = NULL;
+ if (NOT objectName_.isNull()) {
+ objectName = space->allocateAlignedSpace(objectName_.length() + 1);
+ strcpy(objectName, objectName_.data());
+ }
+
// add this descriptor to the work cri descriptor.
#pragma nowarn(1506) // warning elimination
returnedDesc->setTupleDescriptor(returnedDesc->noTuples()-1, tupleDesc);
@@ -4606,25 +4633,25 @@ short HiveMDaccessFunc::codeGen(Generator * generator)
else if (mdType_ == "SYSTEM_TABLES")
type = ComTdbExeUtilHiveMDaccess::SYSTEM_TABLES_;
-#pragma nowarn(1506) // warning elimination
ComTdbExeUtilHiveMDaccess *hiveTdb
= new(space)
ComTdbExeUtilHiveMDaccess(
- type,
- tupleLength,
- givenDesc, // given_cri_desc
- returnedDesc, // returned cri desc
- workCriDesc,
- work_atp_index,
- 8, // Down queue size
- 16, // Up queue size0
- 3, // Number of buffers to allocate
- 36000, // Size of each buffer
- scanExpr, // predicate
- hivePredStr,
- schemaName);
-
-#pragma warn(1506) // warning elimination
+ type,
+ tupleLength,
+ givenDesc, // given_cri_desc
+ returnedDesc, // returned cri desc
+ workCriDesc,
+ work_atp_index,
+ 8, // Down queue size
+ 16, // Up queue size0
+ 3, // Number of buffers to allocate
+ 36000, // Size of each buffer
+ scanExpr, // predicate
+ hivePredStr,
+ catalogName,
+ schemaName,
+ objectName);
+
generator->initTdbFields(hiveTdb);
// Add the explain Information for this node to the EXPLAIN
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/BindRelExpr.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/BindRelExpr.cpp b/core/sql/optimizer/BindRelExpr.cpp
index 7348228..5812a62 100644
--- a/core/sql/optimizer/BindRelExpr.cpp
+++ b/core/sql/optimizer/BindRelExpr.cpp
@@ -1320,6 +1320,14 @@ TrafDesc *generateSpecialDesc(const CorrName& corrName)
ExeUtilRegionStats eudss(TRUE);
desc = eudss.createVirtualTableDesc();
}
+ else if (HiveMDaccessFunc::isHiveMD(corrName.getQualifiedNameObj().getObjectName()))
+ {
+ NAString mdType =
+ HiveMDaccessFunc::getMDType(corrName.getQualifiedNameObj().getObjectName());
+
+ HiveMDaccessFunc hivemd(&mdType);
+ desc = hivemd.createVirtualTableDesc();
+ }
}
return desc;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/ImplRule.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ImplRule.cpp b/core/sql/optimizer/ImplRule.cpp
index dca6b8e..a28a8f6 100644
--- a/core/sql/optimizer/ImplRule.cpp
+++ b/core/sql/optimizer/ImplRule.cpp
@@ -468,7 +468,7 @@ void CreateImplementationRules(RuleSet* set)
r = new(CmpCommon::contextHeap()) PhysicalHiveMDRule
("Implement HiveMDaccessFunc by a PhysicalExplain",
new(CmpCommon::contextHeap())
- HiveMDaccessFunc(NULL, NULL, CmpCommon::contextHeap()),
+ HiveMDaccessFunc(NULL, NULL, NULL, CmpCommon::contextHeap()),
new(CmpCommon::contextHeap())
PhysicalHiveMD(CmpCommon::contextHeap()));
set->insert(r);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index c119f78..a801434 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -87,8 +87,6 @@
#include "CmpSeabaseDDL.h"
#define MAX_NODE_NAME 9
-#define MAX_PRECISION_ALLOWED 18
-#define MAX_NUM_LEN 16
#include "SqlParserGlobals.h"
@@ -3491,189 +3489,7 @@ NABoolean createNAColumns(TrafDesc *column_desc_list /*IN*/,
NAType* getSQColTypeForHive(const char* hiveType, NAMemory* heap)
{
- if ( !strcmp(hiveType, "tinyint"))
- {
- if (CmpCommon::getDefault(TRAF_TINYINT_SUPPORT) == DF_OFF)
- return new (heap) SQLSmall(TRUE /* neg */, TRUE /* allow NULL*/, heap);
- else
- return new (heap) SQLTiny(TRUE /* neg */, TRUE /* allow NULL*/, heap);
- }
-
- if ( !strcmp(hiveType, "smallint"))
- return new (heap) SQLSmall(TRUE /* neg */, TRUE /* allow NULL*/, heap);
-
- if ( !strcmp(hiveType, "int"))
- return new (heap) SQLInt(TRUE /* neg */, TRUE /* allow NULL*/, heap);
-
- if ( !strcmp(hiveType, "bigint"))
- return new (heap) SQLLargeInt(TRUE /* neg */, TRUE /* allow NULL*/, heap);
-
- if ( !strcmp(hiveType, "boolean"))
- return new (heap) SQLBooleanNative(TRUE, heap);
-
- if ( !strcmp(hiveType, "string"))
- {
- Int32 len = CmpCommon::getDefaultLong(HIVE_MAX_STRING_LENGTH);
- Int32 lenInBytes = CmpCommon::getDefaultLong(HIVE_MAX_STRING_LENGTH_IN_BYTES);
- if( lenInBytes != 32000 )
- len = lenInBytes;
- NAString hiveCharset =
- ActiveSchemaDB()->getDefaults().getValue(HIVE_DEFAULT_CHARSET);
- hiveCharset.toUpper();
- CharInfo::CharSet hiveCharsetEnum = CharInfo::getCharSetEnum(hiveCharset);
- Int32 maxNumChars = 0;
- Int32 storageLen = len;
- SQLVarChar * nat =
- new (heap) SQLVarChar(CharLenInfo(maxNumChars, storageLen),
- TRUE, // allow NULL
- FALSE, // not upshifted
- FALSE, // not case-insensitive
- CharInfo::getCharSetEnum(hiveCharset),
- CharInfo::DefaultCollation,
- CharInfo::IMPLICIT);
- nat->setWasHiveString(TRUE);
- return nat;
- }
-
- if ( !strcmp(hiveType, "float"))
- return new (heap) SQLReal(TRUE /* allow NULL*/, heap);
-
- if ( !strcmp(hiveType, "double"))
- return new (heap) SQLDoublePrecision(TRUE /* allow NULL*/, heap);
-
- if ( !strcmp(hiveType, "timestamp"))
- return new (heap) SQLTimestamp(TRUE /* allow NULL */ , 6, heap);
-
- if ( !strcmp(hiveType, "date"))
- return new (heap) SQLDate(TRUE /* allow NULL */ , heap);
-
- if ( !strncmp(hiveType, "varchar", 7) )
- {
- char maxLen[32];
- memset(maxLen, 0, 32);
- int i=0,j=0;
- int copyit = 0;
- int lenStr = strlen(hiveType);
- //get length
- for(i = 0; i < lenStr ; i++)
- {
- if(hiveType[i] == '(') //start
- {
- copyit=1;
- continue;
- }
- else if(hiveType[i] == ')') //stop
- break;
- if(copyit > 0)
- {
- maxLen[j] = hiveType[i];
- j++;
- }
- }
- Int32 len = atoi(maxLen);
-
- if(len == 0) return NULL; //cannot parse correctly
-
- NAString hiveCharset =
- ActiveSchemaDB()->getDefaults().getValue(HIVE_DEFAULT_CHARSET);
-
- hiveCharset.toUpper();
- CharInfo::CharSet hiveCharsetEnum = CharInfo::getCharSetEnum(hiveCharset);
- Int32 maxNumChars = 0;
- Int32 storageLen = len;
- if (CharInfo::isVariableWidthMultiByteCharSet(hiveCharsetEnum))
- {
- // For Hive VARCHARs, the number specified is the max. number of characters,
- // while we count in bytes when using HIVE_MAX_STRING_LENGTH for Hive STRING
- // columns. Set the max character constraint and also adjust the required storage length.
- maxNumChars = len;
- storageLen = len * CharInfo::maxBytesPerChar(hiveCharsetEnum);
- }
- return new (heap) SQLVarChar(CharLenInfo(maxNumChars, storageLen),
- TRUE, // allow NULL
- FALSE, // not upshifted
- FALSE, // not case-insensitive
- CharInfo::getCharSetEnum(hiveCharset),
- CharInfo::DefaultCollation,
- CharInfo::IMPLICIT);
- }
-
- if ( !strncmp(hiveType, "decimal", 7) )
- {
- Int32 i=0, pstart=-1, pend=-1, sstart=-1, send=-1, p=-1, s = -1;
- Int32 hiveTypeLen = strlen(hiveType);
- char pstr[MAX_NUM_LEN], sstr[MAX_NUM_LEN];
- memset(pstr,0,sizeof(pstr));
- memset(sstr,0,sizeof(sstr));
-
- for( i = 0; i < hiveTypeLen; i++ )
- {
- if(hiveType[i] == '(' )
- {
- pstart = i+1;
- }
- else if(hiveType[i] == ',')
- {
- pend = i;
- sstart = i+1;
- }
- else if(hiveType[i] == ')')
- {
- send = i;
- }
- else
- continue;
- }
- if(pend == -1) // no comma found, so no sstart and send
- {
- pend = send;
- send = -1;
- s = 0;
- }
- if(pend - pstart > 0)
- {
- if( (pend - pstart) >= MAX_NUM_LEN ) // too long
- return NULL;
- strncpy(pstr,hiveType+pstart, pend-pstart);
- p=atoi(pstr);
- }
-
- if(send - sstart > 0)
- {
- if( (send - sstart) >= MAX_NUM_LEN ) // too long
- return NULL;
- strncpy(sstr,hiveType+sstart,send-sstart);
- s=atoi(sstr);
- }
-
- if( (p>0) && (p <= MAX_PRECISION_ALLOWED) ) //have precision between 1 - 18
- {
- if( ( s >=0 ) && ( s<= p) ) //have valid scale
- return new (heap) SQLDecimal( p, s, TRUE, TRUE);
- else
- return NULL;
- }
- else if( p > MAX_PRECISION_ALLOWED)
- {
- if ( (s>=0) && ( s<= p ) ) //have valid scale
- return new (heap) SQLBigNum( p, s, TRUE, TRUE, TRUE, NULL);
- else
- return NULL;
- }
- //no p and s given, p and s are all initial value
- else if( ( p == -1 ) && ( s == -1 ) )
- {
- // hive define decimal as decimal ( 10, 0 )
- return new (heap) SQLDecimal( 10, 0, TRUE, TRUE);
- }
- else
- {
- return NULL;
- }
-
- }
-
- return NULL;
+ return NAType::getNATypeForHive(hiveType, heap);
}
NABoolean createNAColumns(struct hive_column_desc* hcolumn /*IN*/,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/RelRoutine.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelRoutine.cpp b/core/sql/optimizer/RelRoutine.cpp
index 4782ab5..990b790 100644
--- a/core/sql/optimizer/RelRoutine.cpp
+++ b/core/sql/optimizer/RelRoutine.cpp
@@ -1357,7 +1357,8 @@ const NAString ExplainFunc::getText() const
// -----------------------------------------------------------------------
HiveMDaccessFunc::HiveMDaccessFunc(NAString *mdt,
- SchemaName* schName,
+ NAString* schName,
+ NAString* objName,
CollHeap *oHeap)
: BuiltinTableValuedFunction(NULL,REL_HIVEMD_ACCESS,oHeap)
{
@@ -1365,6 +1366,8 @@ HiveMDaccessFunc::HiveMDaccessFunc(NAString *mdt,
mdType_ = *mdt;
if (schName)
schemaName_ = *schName;
+ if (objName)
+ objectName_ = *objName;
}
//! HiveMDaccessFunc::~HiveMDaccessFunc Destructor
@@ -1378,13 +1381,13 @@ RelExpr * HiveMDaccessFunc::copyTopNode(RelExpr *derivedNode, CollHeap* outHeap)
HiveMDaccessFunc *result;
if (derivedNode == NULL)
- result = new (outHeap) HiveMDaccessFunc(NULL,NULL,outHeap);
+ result = new (outHeap) HiveMDaccessFunc(NULL,NULL,NULL,outHeap);
else
result = (HiveMDaccessFunc *) derivedNode;
result->mdType_ = mdType_;
result->schemaName_ = schemaName_;
-
+ result->objectName_ = objectName_;
return BuiltinTableValuedFunction::copyTopNode(result, outHeap);
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/RelRoutine.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/RelRoutine.h b/core/sql/optimizer/RelRoutine.h
index 63ea224..438d01e 100644
--- a/core/sql/optimizer/RelRoutine.h
+++ b/core/sql/optimizer/RelRoutine.h
@@ -1382,12 +1382,12 @@ public:
//! HiveMDaccessFunc Constructor
HiveMDaccessFunc(NAString *mdt = NULL,
- SchemaName* schName = NULL,
+ NAString* schName = NULL,
+ NAString* objName = NULL,
CollHeap *oHeap = CmpCommon::statementHeap());
// destructors
-
//! ~HiveMDaccessFunc Destructor
virtual ~HiveMDaccessFunc();
@@ -1400,6 +1400,8 @@ public:
// returns a const char pointer to the name of the virtual Table
// should return a CorrName?##
virtual const char *getVirtualTableName();
+ static NABoolean isHiveMD(const NAString &name);
+ static NAString getMDType(const NAString &name);
//! getArity method
// get the degree of this node (it is a leaf op).
@@ -1456,9 +1458,13 @@ private:
Generator * generator);
NAString mdType_;
+
// Hive schema to be accessed
// If this is passed in as NULL, then the current default is used.
- SchemaName schemaName_;
+ NAString schemaName_;
+
+ // hive object name to be accessed. If null, all objects are returned.
+ NAString objectName_;
}; // class HiveMDaccessFunc
@@ -1479,7 +1485,7 @@ public:
//! PhysicalHiveMD Constructor
PhysicalHiveMD(CollHeap *oHeap = CmpCommon::statementHeap())
- : HiveMDaccessFunc(NULL, NULL, oHeap)
+ : HiveMDaccessFunc(NULL, NULL, NULL, oHeap)
{}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/optimizer/hiveHook.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/hiveHook.h b/core/sql/optimizer/hiveHook.h
index 442b164..bf79f98 100644
--- a/core/sql/optimizer/hiveHook.h
+++ b/core/sql/optimizer/hiveHook.h
@@ -139,6 +139,8 @@ struct hive_sd_desc
char recordTerminator_;
char* nullFormat_;
+ NABoolean isCompressed_;
+
struct hive_sd_desc* next_;
hive_sd_desc(Int32 sdID, const char* loc, Int64 creationTS, Int32 buckets,
@@ -148,13 +150,14 @@ struct hive_sd_desc
struct hive_column_desc* column,
struct hive_skey_desc* skey,
struct hive_bkey_desc* bkey,
- char fieldTerminator, char recordTerminator
+ char fieldTerminator, char recordTerminator,
+ const NABoolean isCompressed
)
-
: sdID_(sdID), buckets_(buckets), kind_(knd), column_(column),
skey_(skey), bkey_(bkey),
fieldTerminator_(fieldTerminator),
recordTerminator_(recordTerminator),
+ isCompressed_(isCompressed),
next_(NULL)
{
location_ = strduph(loc, CmpCommon::contextHeap());
@@ -179,75 +182,82 @@ struct hive_sd_desc
struct hive_tbl_desc
{
Int32 tblID_; // not used with JNI
- char* tblName_;
- char* schName_;
- Int64 creationTS_;
- struct hive_sd_desc* sd_;
- struct hive_pkey_desc* pkey_;
-
- struct hive_tbl_desc* next_;
-
+ char* tblName_;
+ char* schName_;
+ char* owner_;
+ char* tableType_;
+ Int64 creationTS_;
+ struct hive_sd_desc* sd_;
+ struct hive_pkey_desc* pkey_;
+
+ struct hive_tbl_desc* next_;
+
hive_tbl_desc(Int32 tblID, const char* name, const char* schName,
+ const char * owner,
+ const char * tableType,
Int64 creationTS, struct hive_sd_desc* sd,
- struct hive_pkey_desc* pk)
- : tblID_(tblID), sd_(sd), creationTS_(creationTS), pkey_(pk), next_(NULL)
- { tblName_ = strduph(name, CmpCommon::contextHeap());
- schName_ = strduph(schName, CmpCommon::contextHeap()); }
-
- ~hive_tbl_desc();
+ struct hive_pkey_desc* pk);
- struct hive_sd_desc* getSDs() { return sd_; };
+ ~hive_tbl_desc();
- struct hive_skey_desc* getSortKeys();
- struct hive_bkey_desc* getBucketingKeys();
- struct hive_pkey_desc* getPartKey() { return pkey_; };
- struct hive_column_desc* getColumns();
- Int32 getNumOfPartCols() const;
+ struct hive_sd_desc* getSDs() { return sd_; };
+
+ struct hive_skey_desc* getSortKeys();
+ struct hive_bkey_desc* getBucketingKeys();
+ struct hive_pkey_desc* getPartKey() { return pkey_; };
+ struct hive_column_desc* getColumns();
+
+ Int32 getNumOfCols();
+ Int32 getNumOfPartCols() const;
+ Int32 getNumOfSortCols();
+ Int32 getNumOfBucketCols();
+
+ Int32 getPartColNum(const char* name);
+ Int32 getBucketColNum(const char* name);
+ Int32 getSortColNum(const char* name);
- Int64 redeftime();
+ Int64 redeftime();
};
class HiveMetaData
{
public:
- HiveMetaData();
- ~HiveMetaData();
-
- NABoolean init(NABoolean readEntireSchema = FALSE,
- const char * hiveSchemaName = "default",
- const char * tabSearchPredStr = 0);
-
- NABoolean connect();
- NABoolean disconnect();
-
- struct hive_tbl_desc* getTableDesc(const char* schemaName,
- const char* tblName);
- struct hive_tbl_desc* getFakedTableDesc(const char* tblName);
-
- // validate a cached hive table descriptor
+ HiveMetaData();
+ ~HiveMetaData();
+
+ NABoolean init();
+
+ NABoolean connect();
+ NABoolean disconnect();
+
+ struct hive_tbl_desc* getTableDesc(const char* schemaName,
+ const char* tblName);
+ struct hive_tbl_desc* getFakedTableDesc(const char* tblName);
+
+ // validate a cached hive table descriptor
NABoolean validate(Int32 tableId, Int64 redefTS,
const char* schName, const char* tblName);
-
- // iterator over all tables in a Hive schema (default)
- // or iterate over all schemas in the Hive metadata
- void position();
- struct hive_tbl_desc * getNext();
- void advance();
- NABoolean atEnd();
-
- // what the Hive default schema is called in the Hive metadata
+
+ // iterator over all tables in a Hive schema (default)
+ // or iterate over all schemas in the Hive metadata
+ void position();
+ struct hive_tbl_desc * getNext();
+ void advance();
+ NABoolean atEnd();
+
+ // what the Hive default schema is called in the Hive metadata
static const char *getDefaultSchemaName() { return "default"; }
-
- // get lower-level error code
+
+ // get lower-level error code
Int32 getErrCode() const { return errCode_; }
-
+
const char* getErrDetail() const {return errDetail_; }
-
+
const char* getErrCodeStr() const {return errCodeStr_; }
-
+
const char* getErrMethodName() const {return errMethodName_; }
-
+
void resetErrorInfo();
// return TRUE for success, otherwise record error info and return FALSE
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/734783c1/core/sql/parser/sqlparser.y
----------------------------------------------------------------------
diff --git a/core/sql/parser/sqlparser.y b/core/sql/parser/sqlparser.y
index f1cbee2..4d7b730 100755
--- a/core/sql/parser/sqlparser.y
+++ b/core/sql/parser/sqlparser.y
@@ -5896,12 +5896,17 @@ TOK_TABLE '(' TOK_INTERNALSP '(' character_string_literal ')' ')'
}
| TOK_TABLE '(' TOK_HIVEMD '(' hivemd_identifier ')' ')'
{
- $$ = new (PARSERHEAP()) HiveMDaccessFunc($5);
+ $$ = new (PARSERHEAP()) HiveMDaccessFunc($5);
}
-| TOK_TABLE '(' TOK_HIVEMD '(' hivemd_identifier ',' schema_name ')' ')'
+| TOK_TABLE '(' TOK_HIVEMD '(' hivemd_identifier ',' identifier ')' ')'
{
$$ = new (PARSERHEAP()) HiveMDaccessFunc($5, $7);
}
+| TOK_TABLE '(' TOK_HIVEMD '(' hivemd_identifier ',' identifier ',' identifier ')' ')'
+ {
+ $$ = new (PARSERHEAP()) HiveMDaccessFunc($5, $7, $9);
+ }
+
| TOK_TABLE '(' TOK_QUERY_CACHE '(' value_expression_list ')' ')'
{
$$ = new (PARSERHEAP()) RelInternalSP("QUERYCACHE"
@@ -6378,7 +6383,7 @@ index_hint : qualified_name
}
| qualified_guardian_name '.' subvolume_name '.' identifier
{
- NAString *nam = new (PARSERHEAP()) NAString(($1->data()),PARSERHEAP());
+ NAString *nam = new (PARSERHEAP()) NAString(($1->data()),PARSERHEAP());
nam->append(".", 1);
nam->append(($3->data()), $3->length());
nam->append(".", 1);