You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by an...@apache.org on 2016/05/31 22:53:40 UTC
[5/8] incubator-trafodion git commit: Merge remote branch
'origin/master' into ansharma_hivets_br
Merge remote branch 'origin/master' into ansharma_hivets_br
Conflicts:
core/sql/comexe/ComTdbHdfsScan.h
core/sql/generator/GenRelScan.cpp
core/sql/regress/hive/EXPECTED005
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/fa70e683
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/fa70e683
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/fa70e683
Branch: refs/heads/master
Commit: fa70e6831470cccfd86887f9607591f242512fa7
Parents: 1820da1 d199362
Author: Anoop Sharma <an...@esgyn.com>
Authored: Sat May 28 01:19:42 2016 +0000
Committer: Anoop Sharma <an...@esgyn.com>
Committed: Sat May 28 01:19:42 2016 +0000
----------------------------------------------------------------------
Makefile | 1 +
core/Makefile | 12 +-
core/conn/jdbcT4/pom.xml | 7 +-
.../jdbcT4/src/main/java/T4Messages.properties | 4 +-
core/conn/jdbc_type2/build.xml | 11 +-
core/conn/odbc/src/odbc/Makefile | 8 +-
core/conn/trafci/install/Installer.java | 7 +-
.../odbc/odbcclient/unixcli/makefile.lnx | 6 +-
.../odbc/odbcclient/unixcli/package/TRAFDSN | 2 +-
.../odbc/odbcclient/unixcli/package/mklnxpkg.sh | 2 +
.../unixcli/package/runconnect_test.sh | 26 +
core/dbsecurity/scripts/sqcertgen | 5 -
core/dbsecurity/scripts/update_auth | 9 +-
core/sqf/Makefile | 2 +
core/sqf/samples/.gitignore | 4 +
core/sqf/sql/scripts/dcscheck | 20 +-
core/sqf/sql/scripts/install_traf_components | 3 +-
core/sqf/sql/scripts/sqcheck | 57 +-
core/sqf/sqvers | 2 +-
core/sql/comexe/ComTdbFastTransport.h | 4 +-
core/sql/comexe/ComTdbHdfsScan.cpp | 13 +-
core/sql/comexe/ComTdbHdfsScan.h | 56 +-
core/sql/common/ComSmallDefs.h | 3 +
core/sql/executor/ExFastTransport.cpp | 18 +-
core/sql/executor/ExHdfsScan.cpp | 38 +-
core/sql/executor/hiveHook.cpp | 27 +-
core/sql/generator/GenFastTransport.cpp | 5 +-
core/sql/generator/GenRelScan.cpp | 32 +-
core/sql/lib_mgmt/pom.xml | 2 +-
core/sql/optimizer/BindRelExpr.cpp | 30 +-
core/sql/optimizer/HDFSHook.cpp | 1 +
core/sql/optimizer/HDFSHook.h | 4 +
core/sql/optimizer/RelFastTransport.cpp | 1 +
core/sql/optimizer/RelFastTransport.h | 18 +-
core/sql/optimizer/hiveHook.h | 25 +-
core/sql/regress/core/EXPECTED162 | 22 +-
core/sql/regress/hive/EXPECTED001 | 22 +-
core/sql/regress/hive/EXPECTED003 | 244 ++++----
core/sql/regress/hive/EXPECTED004 | 240 ++++----
core/sql/regress/hive/EXPECTED005 | 10 +-
core/sql/regress/hive/EXPECTED006 | 16 +-
core/sql/regress/hive/EXPECTED015 | 64 +--
core/sql/regress/hive/EXPECTED018 | 570 +++++++++++--------
core/sql/regress/hive/TEST018 | 48 +-
.../hive/TEST018_create_hive_tables.hive | 16 +-
core/sql/regress/tools/runregr_hive.ksh | 39 +-
core/sql/sqlcomp/DefaultConstants.h | 1 -
core/sql/sqlcomp/nadefaults.cpp | 1 -
core/sql/ustat/hs_globals.cpp | 28 +-
dcs/pom.xml | 8 +
install/Makefile | 18 +-
install/installer/traf_cloudera_mods | 1 -
install/installer/traf_package_setup | 2 +-
53 files changed, 1077 insertions(+), 738 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/comexe/ComTdbHdfsScan.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/comexe/ComTdbHdfsScan.h
----------------------------------------------------------------------
diff --cc core/sql/comexe/ComTdbHdfsScan.h
index c60b192,70573e1..ac83311
--- a/core/sql/comexe/ComTdbHdfsScan.h
+++ b/core/sql/comexe/ComTdbHdfsScan.h
@@@ -131,18 -131,11 +131,18 @@@ class ComTdbHdfsScan : public ComTd
NABasicPtr loggingLocation_; // 168 - 175
NABasicPtr errCountRowId_; // 176 - 183
UInt32 hiveScanMode_; // 184 - 187
-
- char fillersComTdbHdfsScan1_[4]; // 188 - 191
+ UInt16 origTuppIndex_; // 188 - 189
+ char fillersComTdbHdfsScan1_[2]; // 190 - 191
+ NABasicPtr nullFormat_; // 192 - 199
- char fillersComTdbHdfsScan2_[8]; // 200 - 207
+ // next 3 params used to check if data under hdfsFileDir
+ // was modified after query was compiled.
- NABasicPtr hdfsRootDir_; // 192 - 199
- Int64 modTSforDir_; // 200 - 207
- Lng32 numOfPartCols_; // 208 - 211
- QueuePtr hdfsDirsToCheck_; // 212 - 219
-
++ NABasicPtr hdfsRootDir_; // 200 - 207
++ Int64 modTSforDir_; // 208 - 215
++ Lng32 numOfPartCols_; // 216 - 219
+ char fillersComTdbHdfsScan2_[4]; // 220 - 223
++ QueuePtr hdfsDirsToCheck_; // 224 - 231
+
public:
enum HDFSFileType
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/generator/GenRelScan.cpp
----------------------------------------------------------------------
diff --cc core/sql/generator/GenRelScan.cpp
index 5474d94,9c17fad..d1cb243
--- a/core/sql/generator/GenRelScan.cpp
+++ b/core/sql/generator/GenRelScan.cpp
@@@ -1146,32 -1178,15 +1165,41 @@@ if (hTabStats->isOrcFile()
char * tablename =
space->AllocateAndCopyToAlignedSpace(GenGetQualifiedName(getIndexDesc()->getNAFileSet()->getFileSetName()), 0);
+ char * nullFormat = NULL;
+ if (hTabStats->getNullFormat())
+ {
+ nullFormat =
+ space->allocateAndCopyToAlignedSpace(hTabStats->getNullFormat(),
+ strlen(hTabStats->getNullFormat()),
+ 0);
+ }
+
+ // info needed to validate hdfs file structs
+ char * hdfsRootDir = NULL;
+ Int64 modTS = -1;
+ Lng32 numOfPartLevels = -1;
+ Queue * hdfsDirsToCheck = NULL;
+ if (CmpCommon::getDefault(HIVE_DATA_MOD_CHECK) == DF_ON)
+ {
+ hdfsRootDir =
+ space->allocateAndCopyToAlignedSpace(hTabStats->tableDir().data(),
+ hTabStats->tableDir().length(),
+ 0);
+ modTS = hTabStats->getModificationTS();
+ numOfPartLevels = hTabStats->numOfPartCols();
+
+ // if specific directories are to checked based on the query struct
+ // (for example, when certain partitions are explicitly specified),
+ // add them to hdfsDirsToCheck.
+ // At runtime, only these dirs will be checked for data modification.
+ // ** TBD **
+
+ // Right now, timestamp info is not being generated correctly for
+ // partitioned files. Skip data mod check for them.
+ if (numOfPartLevels > 0)
+ hdfsRootDir = NULL;
+ }
+
// create hdfsscan_tdb
ComTdbHdfsScan *hdfsscan_tdb = new(space)
ComTdbHdfsScan(
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/optimizer/HDFSHook.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/optimizer/HDFSHook.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/regress/hive/EXPECTED005
----------------------------------------------------------------------
diff --cc core/sql/regress/hive/EXPECTED005
index 6118229,28d4f37..3c286d6
--- a/core/sql/regress/hive/EXPECTED005
+++ b/core/sql/regress/hive/EXPECTED005
@@@ -226,9 -221,9 +226,9 @@@ xy
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
- N 19631
- 685
-N 9789
-Y 9525
++ 19631
+Y 18984
+? 1384
--- 3 row(s) selected.
>>execute s3;
@@@ -316,9 -307,9 +316,9 @@@ A
C_PREFERRED_CUST_FLAG (EXPR)
------------------------- --------------------
- N 19631
- 685
-N 9789
-Y 9525
++ 19631
+Y 18984
+? 1384
--- 3 row(s) selected.
>>execute s4;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/fa70e683/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------