You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2016/12/19 23:18:58 UTC
[1/8] incubator-trafodion git commit: Merge branch 'trafodion-2356'
of github.com:selvaganesang/incubator-trafodion into trafodion-2356
Repository: incubator-trafodion
Updated Branches:
refs/heads/master ac483c91e -> c5568f2a9
Merge branch 'trafodion-2356' of github.com:selvaganesang/incubator-trafodion into trafodion-2356
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/055f7bb7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/055f7bb7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/055f7bb7
Branch: refs/heads/master
Commit: 055f7bb7ba344c7273e7ce2e725295a29f8768a2
Parents: 10e4bc8 7aabbab
Author: selvaganesang <se...@esgyn.com>
Authored: Fri Nov 18 22:12:57 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Fri Nov 18 22:12:57 2016 +0000
----------------------------------------------------------------------
core/sql/executor/HBaseClient_JNI.cpp | 666 ++++++-------------------
core/sql/executor/JavaObjectInterface.cpp | 14 +
core/sql/executor/JavaObjectInterface.h | 4 +-
3 files changed, 176 insertions(+), 508 deletions(-)
----------------------------------------------------------------------
[8/8] incubator-trafodion git commit: Merge [TRAFODION-2351] PR 876
Bulk load with log error rows enhancements
Posted by se...@apache.org.
Merge [TRAFODION-2351] PR 876 Bulk load with log error rows enhancements
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c5568f2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c5568f2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c5568f2a
Branch: refs/heads/master
Commit: c5568f2a962ae0bda9f75bb919aca7bce97b0f62
Parents: ac483c9 884eee0
Author: selvaganesang <se...@apache.org>
Authored: Mon Dec 19 23:17:21 2016 +0000
Committer: selvaganesang <se...@apache.org>
Committed: Mon Dec 19 23:17:21 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 10 +-
core/sql/executor/ExExeUtilCommon.cpp | 25 +-
core/sql/executor/ExExeUtilLoad.cpp | 160 ++++++---
core/sql/executor/ExHbaseAccess.cpp | 27 +-
core/sql/executor/ExHbaseAccess.h | 6 +-
core/sql/executor/ExHbaseIUD.cpp | 9 +-
core/sql/executor/ExHdfsScan.cpp | 9 +-
core/sql/executor/ExHdfsScan.h | 2 +-
core/sql/regress/executor/EXPECTED140 | 61 ++--
core/sql/regress/executor/FILTER140 | 1 +
core/sql/regress/hive/EXPECTED015 | 230 +++++++------
core/sql/regress/hive/EXPECTED018 | 512 +++++++++++++++++------------
core/sql/regress/hive/FILTER015 | 3 +-
core/sql/regress/hive/FILTER018 | 3 +-
core/sql/sqlcomp/nadefaults.cpp | 2 +-
15 files changed, 660 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c5568f2a/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c5568f2a/core/sql/executor/ExHbaseAccess.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c5568f2a/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c5568f2a/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c5568f2a/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
[4/8] incubator-trafodion git commit: Merge branch 'master' of
git://git.apache.org/incubator-trafodion into trafodion-2351
Posted by se...@apache.org.
Merge branch 'master' of git://git.apache.org/incubator-trafodion into trafodion-2351
Conflicts:
core/sql/executor/ExExeUtilLoad.cpp
core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/76cb8b4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/76cb8b4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/76cb8b4b
Branch: refs/heads/master
Commit: 76cb8b4bf3f4f99acdeebb4f01c722af4079f1ff
Parents: 1c35622 515ff4c
Author: selvaganesang <se...@esgyn.com>
Authored: Thu Dec 8 21:32:53 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Thu Dec 8 21:32:53 2016 +0000
----------------------------------------------------------------------
.rat-excludes | 3 +-
core/Makefile | 6 +-
core/conn/jdbcT4/pom.xml | 79 +-
.../trafodion/jdbc/t4/InterfaceUtilities.java | 3 +-
core/conn/odbc/src/odbc/Common/QSData.cpp | 17 +
core/conn/odbc/src/odbc/Common/QSData.h | 33 +
core/conn/odbc/src/odbc/nsksrvrcore/Makefile | 2 +-
core/conn/trafci/.gitignore | 6 +
core/conn/trafci/Makefile | 37 +
core/conn/trafci/install/Installer.java | 3389 --------------
core/conn/trafci/install/Installer.java-tmpl | 3389 ++++++++++++++
core/conn/trafci/installer_pom.xml | 106 +
core/conn/trafci/pom.xml | 114 +
.../org/trafodion/ci/AWTExceptionHandler.java | 36 +
.../main/java/org/trafodion/ci/CSVDisplay.java | 90 +
.../src/main/java/org/trafodion/ci/Column.java | 131 +
.../java/org/trafodion/ci/ConditionalQuery.java | 435 ++
.../trafodion/ci/ConditionalQueryException.java | 43 +
.../trafodion/ci/ConditionalSyntaxError.java | 123 +
.../java/org/trafodion/ci/ConsoleReader.java | 218 +
.../java/org/trafodion/ci/ConsoleWriter.java | 130 +
.../java/org/trafodion/ci/DatabaseQuery.java | 1804 ++++++++
.../src/main/java/org/trafodion/ci/Display.java | 68 +
.../trafodion/ci/DuplicateParamException.java | 40 +
.../main/java/org/trafodion/ci/ErrorObject.java | 103 +
.../src/main/java/org/trafodion/ci/FCQuery.java | 334 ++
.../main/java/org/trafodion/ci/FileReader.java | 176 +
.../main/java/org/trafodion/ci/FileWriter.java | 80 +
.../main/java/org/trafodion/ci/HTMLDisplay.java | 80 +
.../main/java/org/trafodion/ci/HTMLObject.java | 628 +++
.../main/java/org/trafodion/ci/HelpIndexer.java | 134 +
.../src/main/java/org/trafodion/ci/Helper.java | 225 +
.../main/java/org/trafodion/ci/HostQuery.java | 289 ++
.../main/java/org/trafodion/ci/IDisplay.java | 28 +
.../main/java/org/trafodion/ci/IndexStruct.java | 55 +
.../java/org/trafodion/ci/InterfaceQuery.java | 4378 ++++++++++++++++++
.../trafodion/ci/InterfaceQueryException.java | 35 +
.../org/trafodion/ci/InterfaceSyntaxError.java | 83 +
.../trafodion/ci/InvalidNumberOfArguments.java | 35 +
.../main/java/org/trafodion/ci/JDBCVproc.java | 38 +
.../java/org/trafodion/ci/LFProperties.java | 130 +
.../java/org/trafodion/ci/MaskingThread.java | 73 +
.../java/org/trafodion/ci/MultiByteUtils.java | 56 +
.../org/trafodion/ci/NullKeyWordException.java | 35 +
.../org/trafodion/ci/OutputColumnAttribute.java | 47 +
.../java/org/trafodion/ci/OutputContents.java | 144 +
.../main/java/org/trafodion/ci/ParallelRun.java | 1320 ++++++
.../org/trafodion/ci/ParamStringObject.java | 256 +
.../java/org/trafodion/ci/ParseArguments.java | 660 +++
.../src/main/java/org/trafodion/ci/Parser.java | 1195 +++++
.../ci/ParserMissingParamException.java | 42 +
.../trafciDefaultLookAndFeel.properties | 60 +
.../main/java/org/trafodion/ci/PrunSummary.java | 119 +
.../org/trafodion/ci/PrunUserInterruption.java | 34 +
.../java/org/trafodion/ci/PythonInterface.java | 180 +
.../src/main/java/org/trafodion/ci/Query.java | 188 +
.../main/java/org/trafodion/ci/QueryUtils.java | 37 +
.../java/org/trafodion/ci/QueryWrapper.java | 700 +++
.../main/java/org/trafodion/ci/RawDisplay.java | 189 +
.../src/main/java/org/trafodion/ci/Reader.java | 286 ++
.../java/org/trafodion/ci/RepObjInterface.java | 27 +
.../java/org/trafodion/ci/ReplacementObj.java | 39 +
.../org/trafodion/ci/ScriptSectionNotFound.java | 34 +
.../java/org/trafodion/ci/ScriptsInterface.java | 153 +
.../src/main/java/org/trafodion/ci/Session.java | 1908 ++++++++
.../java/org/trafodion/ci/SessionDefaults.java | 342 ++
.../java/org/trafodion/ci/SessionError.java | 181 +
.../java/org/trafodion/ci/SessionInterface.java | 1487 ++++++
.../org/trafodion/ci/SessionTimeoutTask.java | 93 +
.../src/main/java/org/trafodion/ci/SqlEnum.java | 83 +
.../java/org/trafodion/ci/StatusThread.java | 100 +
.../org/trafodion/ci/TerminalProcessBridge.java | 246 +
.../trafodion/ci/UnKnownInterfaceCommand.java | 34 +
.../java/org/trafodion/ci/UserInterface.java | 421 ++
.../java/org/trafodion/ci/UserInterruption.java | 36 +
.../src/main/java/org/trafodion/ci/Utils.java | 381 ++
.../java/org/trafodion/ci/ValidateQuery.java | 412 ++
.../org/trafodion/ci/ValidationException.java | 39 +
.../main/java/org/trafodion/ci/Vproc.java-tmpl | 36 +
.../main/java/org/trafodion/ci/WCIUtils.java | 122 +
.../src/main/java/org/trafodion/ci/Writer.java | 418 ++
.../main/java/org/trafodion/ci/XMLDisplay.java | 75 +
.../main/java/org/trafodion/ci/XMLObject.java | 607 +++
.../org/trafodion/ci/help/common_common.help | 457 ++
.../java/org/trafodion/ci/help/help_file.idx | 30 +
.../java/org/trafodion/ci/help/sql_common.help | 376 ++
.../org/trafodion/ci/pwdencrypt/ArgMap.java | 61 +
.../org/trafodion/ci/pwdencrypt/ArgsDef.java | 74 +
.../trafodion/ci/pwdencrypt/EncryptUtil.java | 249 +
.../java/org/trafodion/ci/pwdencrypt/JCE.java | 414 ++
.../trafodion/ci/pwdencrypt/PasswordOpts.java | 292 ++
.../org/trafodion/ci/pwdencrypt/RefLookup.java | 127 +
.../org/trafodion/ci/pwdencrypt/WorkTypes.java | 55 +
.../trafci/src/main/resources/install_jar.xml | 71 +
.../trafci/src/main/resources/install_tar.xml | 51 +
.../trafci/src/main/resources/trafci.properties | 5 +
.../org/trafodion/ci/AWTExceptionHandler.java | 36 -
.../trafci/src/org/trafodion/ci/CSVDisplay.java | 90 -
.../trafci/src/org/trafodion/ci/Column.java | 131 -
.../src/org/trafodion/ci/ConditionalQuery.java | 435 --
.../trafodion/ci/ConditionalQueryException.java | 43 -
.../trafodion/ci/ConditionalSyntaxError.java | 123 -
.../src/org/trafodion/ci/ConsoleReader.java | 218 -
.../src/org/trafodion/ci/ConsoleWriter.java | 130 -
.../src/org/trafodion/ci/DatabaseQuery.java | 1804 --------
.../trafci/src/org/trafodion/ci/Display.java | 68 -
.../trafodion/ci/DuplicateParamException.java | 40 -
.../src/org/trafodion/ci/ErrorObject.java | 103 -
.../trafci/src/org/trafodion/ci/FCQuery.java | 334 --
.../trafci/src/org/trafodion/ci/FileReader.java | 176 -
.../trafci/src/org/trafodion/ci/FileWriter.java | 80 -
.../src/org/trafodion/ci/HTMLDisplay.java | 80 -
.../trafci/src/org/trafodion/ci/HTMLObject.java | 628 ---
.../src/org/trafodion/ci/HelpIndexer.java | 134 -
.../trafci/src/org/trafodion/ci/Helper.java | 225 -
.../trafci/src/org/trafodion/ci/HostQuery.java | 289 --
.../trafci/src/org/trafodion/ci/IDisplay.java | 28 -
.../src/org/trafodion/ci/IndexStruct.java | 55 -
.../src/org/trafodion/ci/InterfaceQuery.java | 4378 ------------------
.../trafodion/ci/InterfaceQueryException.java | 35 -
.../org/trafodion/ci/InterfaceSyntaxError.java | 83 -
.../trafodion/ci/InvalidNumberOfArguments.java | 35 -
.../trafci/src/org/trafodion/ci/JDBCVproc.java | 38 -
.../src/org/trafodion/ci/LFProperties.java | 130 -
.../src/org/trafodion/ci/MaskingThread.java | 73 -
.../src/org/trafodion/ci/MultiByteUtils.java | 56 -
.../org/trafodion/ci/NullKeyWordException.java | 35 -
.../org/trafodion/ci/OutputColumnAttribute.java | 47 -
.../src/org/trafodion/ci/OutputContents.java | 144 -
.../src/org/trafodion/ci/ParallelRun.java | 1320 ------
.../src/org/trafodion/ci/ParamStringObject.java | 256 -
.../src/org/trafodion/ci/ParseArguments.java | 660 ---
.../trafci/src/org/trafodion/ci/Parser.java | 1195 -----
.../ci/ParserMissingParamException.java | 42 -
.../trafciDefaultLookAndFeel.properties | 60 -
.../src/org/trafodion/ci/PrunSummary.java | 119 -
.../org/trafodion/ci/PrunUserInterruption.java | 34 -
.../src/org/trafodion/ci/PythonInterface.java | 180 -
.../conn/trafci/src/org/trafodion/ci/Query.java | 188 -
.../trafci/src/org/trafodion/ci/QueryUtils.java | 37 -
.../src/org/trafodion/ci/QueryWrapper.java | 700 ---
.../trafci/src/org/trafodion/ci/RawDisplay.java | 189 -
.../trafci/src/org/trafodion/ci/Reader.java | 286 --
.../src/org/trafodion/ci/RepObjInterface.java | 27 -
.../src/org/trafodion/ci/ReplacementObj.java | 39 -
.../org/trafodion/ci/ScriptSectionNotFound.java | 34 -
.../src/org/trafodion/ci/ScriptsInterface.java | 153 -
.../trafci/src/org/trafodion/ci/Session.java | 1908 --------
.../src/org/trafodion/ci/SessionDefaults.java | 342 --
.../src/org/trafodion/ci/SessionError.java | 181 -
.../src/org/trafodion/ci/SessionInterface.java | 1487 ------
.../org/trafodion/ci/SessionTimeoutTask.java | 93 -
.../trafci/src/org/trafodion/ci/SqlEnum.java | 83 -
.../src/org/trafodion/ci/StatusThread.java | 100 -
.../org/trafodion/ci/TerminalProcessBridge.java | 246 -
.../trafodion/ci/UnKnownInterfaceCommand.java | 34 -
.../src/org/trafodion/ci/UserInterface.java | 421 --
.../src/org/trafodion/ci/UserInterruption.java | 36 -
.../conn/trafci/src/org/trafodion/ci/Utils.java | 381 --
.../src/org/trafodion/ci/ValidateQuery.java | 412 --
.../org/trafodion/ci/ValidationException.java | 39 -
.../conn/trafci/src/org/trafodion/ci/Vproc.java | 36 -
.../trafci/src/org/trafodion/ci/WCIUtils.java | 122 -
.../trafci/src/org/trafodion/ci/Writer.java | 418 --
.../trafci/src/org/trafodion/ci/XMLDisplay.java | 75 -
.../trafci/src/org/trafodion/ci/XMLObject.java | 607 ---
.../org/trafodion/ci/help/common_common.help | 457 --
.../src/org/trafodion/ci/help/help_file.idx | 30 -
.../src/org/trafodion/ci/help/sql_common.help | 376 --
.../src/org/trafodion/ci/pwdencrypt/ArgMap.java | 61 -
.../org/trafodion/ci/pwdencrypt/ArgsDef.java | 74 -
.../trafodion/ci/pwdencrypt/EncryptUtil.java | 249 -
.../src/org/trafodion/ci/pwdencrypt/JCE.java | 414 --
.../trafodion/ci/pwdencrypt/PasswordOpts.java | 292 --
.../org/trafodion/ci/pwdencrypt/RefLookup.java | 127 -
.../org/trafodion/ci/pwdencrypt/WorkTypes.java | 55 -
.../org/trafodion/ci/UserInterfaceTest.java | 116 +
core/dbsecurity/auth/Makefile | 1 +
core/sqf/.gitignore | 1 +
core/sqf/Makefile | 4 +-
core/sqf/build-scripts/build.branch | 8 +-
core/sqf/build-scripts/build.id | 2 +-
core/sqf/build-scripts/genverhdr.ksh | 15 +-
core/sqf/export/include/dtm/tm.h | 16 +-
core/sqf/export/include/dtm/tmtransaction.h | 13 +-
core/sqf/sql/scripts/analyzeMessageGuide.py | 656 +++
core/sqf/src/seatrans/hbase-trx/pp.awk | 152 +-
.../hbase/client/transactional/RMInterface.java | 40 +-
.../sqf/src/seatrans/tm/hbasetmlib2/hbasetm.cpp | 246 +-
core/sqf/src/seatrans/tm/hbasetmlib2/hbasetm.h | 14 +-
.../tm/hbasetmlib2/javaobjectinterfacetm.cpp | 103 +-
.../tm/hbasetmlib2/javaobjectinterfacetm.h | 11 +-
core/sqf/src/tm/macros.gmk | 2 +-
core/sqf/src/tm/tmddlrequests.cpp | 292 +-
core/sqf/src/tm/tmddlrequests.h | 8 +-
core/sqf/src/tm/tmlib.cpp | 26 +-
core/sqf/src/tm/tmlibmsg.h | 3 +
core/sqf/src/tm/tmrmhbase.cpp | 23 +-
core/sqf/src/tm/tmtransaction.cpp | 52 +-
core/sql/bin/SqlciErrors.txt | 36 +-
core/sql/comexe/ComTdbExeUtil.cpp | 26 +-
core/sql/comexe/ComTdbExeUtil.h | 112 +-
core/sql/common/ComJSON.cpp | 1175 +++++
core/sql/common/ComJSON.h | 159 +
core/sql/common/ComJSONFuncs.cpp | 793 ++++
core/sql/common/ComJSONStringInfo.cpp | 297 ++
core/sql/common/ComJSONStringInfo.h | 176 +
core/sql/common/ComSmallDefs.h | 8 +
core/sql/common/NAString.cpp | 1 +
core/sql/common/NAType.cpp | 201 +
core/sql/common/NAType.h | 2 +
core/sql/common/OperTypeEnum.h | 10 +-
core/sql/executor/ExExeUtil.h | 7 +-
core/sql/executor/ExExeUtilGet.cpp | 365 +-
core/sql/executor/ExExeUtilLoad.cpp | 4 +-
core/sql/executor/hiveHook.cpp | 168 +-
core/sql/exp/ExpErrorEnums.h | 16 +
core/sql/exp/ExpLOBaccess.cpp | 2 -
core/sql/exp/ExpPackDefs.cpp | 16 +
core/sql/exp/exp_clause.cpp | 44 +-
core/sql/exp/exp_clause.h | 5 +-
core/sql/exp/exp_clause_derived.h | 169 +
core/sql/exp/exp_function.cpp | 309 +-
core/sql/exp/exp_function.h | 57 +-
core/sql/exp/exp_like.cpp | 80 +
core/sql/exp/exp_math_func.cpp | 19 +-
core/sql/generator/GenItemFunc.cpp | 49 +-
core/sql/generator/GenRelExeUtil.cpp | 77 +-
core/sql/nskgmake/common/Makefile | 3 +
core/sql/optimizer/BindItemExpr.cpp | 29 +-
core/sql/optimizer/BindRelExpr.cpp | 17 +-
core/sql/optimizer/ImplRule.cpp | 2 +-
core/sql/optimizer/ItemCache.cpp | 4 +-
core/sql/optimizer/ItemExpr.cpp | 59 +-
core/sql/optimizer/ItemFunc.h | 104 +-
core/sql/optimizer/NATable.cpp | 186 +-
core/sql/optimizer/NormItemExpr.cpp | 2 +-
core/sql/optimizer/RelRoutine.cpp | 9 +-
core/sql/optimizer/RelRoutine.h | 14 +-
core/sql/optimizer/SynthType.cpp | 91 +-
core/sql/optimizer/hiveHook.h | 116 +-
core/sql/parser/ParKeyWords.cpp | 5 +-
core/sql/parser/sqlparser.y | 143 +-
core/sql/regress/compGeneral/EXPECTED006.SB | 196 +-
core/sql/regress/compGeneral/TEST006 | 47 +
core/sql/regress/executor/EXPECTED002.SB | 99 +-
core/sql/regress/executor/TEST002 | 29 +-
core/sql/regress/privs2/EXPECTED129 | 16 +-
core/sql/regress/seabase/EXPECTED002 | 266 +-
core/sql/regress/seabase/EXPECTED012 | 52 +-
core/sql/regress/seabase/TEST002 | 28 +
core/sql/regress/seabase/TEST012 | 11 +
core/sql/regress/udr/EXPECTED100.SB | 2 +-
core/sql/sqlcomp/CmpDDLCatErrorCodes.h | 16 +-
core/sql/sqlcomp/CmpSeabaseDDLrepos.cpp | 19 +-
core/sql/sqlcomp/CmpSeabaseDDLtable.cpp | 60 +-
.../java/org/trafodion/sql/HBaseClient.java | 9 +-
core/sql/ustat/hs_cli.cpp | 361 +-
core/sql/ustat/hs_cli.h | 67 +-
core/sql/ustat/hs_globals.cpp | 135 +-
core/sql/ustat/hs_log.h | 15 +-
core/sql/ustat/hs_read.cpp | 4 +-
core/sql/ustat/hs_util.h | 3 +-
.../org/trafodion/jdbc_test/TestNumeric.java | 114 +
dcs/src/test/odbc/README.txt | 47 +
dcs/src/test/odbc/build/Makefile | 43 +
dcs/src/test/odbc/build/odbc.vcxproj | 157 +
dcs/src/test/odbc/build/odbc.vcxproj.filters | 40 +
dcs/src/test/odbc/src/Main.cpp | 57 +
dcs/src/test/odbc/src/common/Global.cpp | 116 +
dcs/src/test/odbc/src/common/Global.h | 60 +
dcs/src/test/odbc/src/common/TestBase.cpp | 263 ++
dcs/src/test/odbc/src/common/TestBase.h | 55 +
.../src/test/varchar32k/LargeVarcharCol.cpp | 2639 +++++++++++
.../odbc/src/test/varchar32k/LargeVarcharCol.h | 52 +
.../src/asciidoc/_chapters/ddl_msgs.adoc | 263 +-
.../sql_functions_and_expressions.adoc | 249 +-
.../_chapters/sql_language_elements.adoc | 3 +-
.../src/asciidoc/_chapters/sql_statements.adoc | 243 +-
licenses/lic-server-src | 29 +-
280 files changed, 40129 insertions(+), 29976 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/76cb8b4b/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --cc core/sql/executor/ExExeUtilLoad.cpp
index 73e0d2f,9f3871d..aea946b
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@@ -2065,10 -2065,9 +2065,10 @@@ short ExExeUtilHBaseBulkUnLoadTcb::getT
short ExExeUtilHBaseBulkUnLoadTcb::work()
{
Lng32 cliRC = 0;
- short retcode = 0;
+ Lng32 retcode = 0;
short rc;
SFW_RetCode sfwRetCode = SFW_OK;
+ Lng32 hbcRetCode = HBC_OK;
// if no parent request, return
if (qparent_.down->isEmpty())
return WORK_OK;
[5/8] incubator-trafodion git commit: [TRAFODION-2351] Bulk load with
log error rows enhancements
Posted by se...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/hive/EXPECTED018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED018 b/core/sql/regress/hive/EXPECTED018
index 56cb05b..d412e5c 100644
--- a/core/sql/regress/hive/EXPECTED018
+++ b/core/sql/regress/hive/EXPECTED018
@@ -145,14 +145,17 @@
>>--load1
>>load with no recovery into customer_address
+>select * from hive.hive.customer_address;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
+Task: CLEANUP Status: Started Time: 2016-12-11 21:38:15.573
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:38:15.610
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.037
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:38:15.610
Rows Processed: 50000
-Task: PREPARATION Status: Ended ET: 00:00:09.197
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: COMPLETION Status: Ended ET: 00:00:00.295
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:38:21.856
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:06.247
+Task: COMPLETION Status: Started Time: 2016-12-11 21:38:21.856
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:38:22.132
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.276
--- 50000 row(s) loaded.
>>--
@@ -176,14 +179,17 @@ Task: COMPLETION Status: Ended ET: 00:00:00.295
>>--load2
>>load with no recovery into customer_demographics
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 20000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: CLEANUP Status: Started Time: 2016-12-11 21:38:24.128
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:38:24.339
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.021
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:38:24.339
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:12.499
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:00.361
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:38:31.722
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:07.689
+Task: COMPLETION Status: Started Time: 2016-12-11 21:38:31.722
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:38:32.127
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.290
--- 20000 row(s) loaded.
>>--
@@ -208,14 +214,17 @@ Task: COMPLETION Status: Ended ET: 00:00:00.361
>>--load3
>>load with no recovery into customer_demographics_salt
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 20000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: CLEANUP Status: Started Time: 2016-12-11 21:38:34.252
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:38:34.274
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.021
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:38:34.274
Rows Processed: 20000
-Task: PREPARATION Status: Ended ET: 00:00:10.844
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.373
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:38:38.597
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:03.786
+Task: COMPLETION Status: Started Time: 2016-12-11 21:38:38.597
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:38:38.312
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.253
--- 20000 row(s) loaded.
>>--
@@ -230,14 +239,17 @@ Task: COMPLETION Status: Ended ET: 00:00:00.373
>>--load4
>>load with no recovery into customer_salt
+>select * from hive.hive.customer;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
+Task: CLEANUP Status: Started Time: 2016-12-11 21:38:39.852
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:38:39.873
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.021
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:38:39.873
Rows Processed: 100000
-Task: PREPARATION Status: Ended ET: 00:00:11.649
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.301
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:38:44.757
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:04.883
+Task: COMPLETION Status: Started Time: 2016-12-11 21:38:44.757
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:38:45.124
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.255
--- 100000 row(s) loaded.
>>--
@@ -261,14 +273,17 @@ Task: COMPLETION Status: Ended ET: 00:00:00.301
>>--load5
>>load with no recovery into store_sales_salt
+>select * from hive.hive.store_sales where ss_item_sk <= 1000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
+Task: LOAD Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
+Task: CLEANUP Status: Started Time: 2016-12-11 21:38:46.752
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:38:46.774
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.021
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:38:46.774
Rows Processed: 160756
-Task: PREPARATION Status: Ended ET: 00:00:35.420
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.STORE_SALES_SALT
-Task: COMPLETION Status: Ended ET: 00:00:00.590
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:38:52.177
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:05.403
+Task: COMPLETION Status: Started Time: 2016-12-11 21:38:52.177
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:38:52.452
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.275
--- 160756 row(s) loaded.
>>--
@@ -355,11 +370,13 @@ a
+> into '/user/hive/exttables/null_format_default'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:38:57.296
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: EXTRACT Status: Started Time: 2016-12-11 21:38:57.296
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.254
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:38:57.347
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.051
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_default;
@@ -384,11 +401,13 @@ a
+> into '/user/hive/exttables/null_format_empty'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:38:58.874
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2016-12-11 21:38:58.874
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.120
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:38:58.517
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.043
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_empty;
@@ -413,11 +432,13 @@ a ?
+> into '/user/hive/exttables/null_format_colon'
+> select * from null_format_src;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:38:59.476
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: EXTRACT Status: Started Time: 2016-12-11 21:38:59.476
Rows Processed: 10
-Task: EXTRACT Status: Ended ET: 00:00:00.289
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:38:59.519
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.042
--- 10 row(s) unloaded.
>>select * from hive.hive.null_format_colon;
@@ -479,13 +500,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_address
+>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.013
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:39:44.997
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:39:44.997
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:01.506
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.121
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:39:45.665
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.668
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:39:45.665
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:39:45.710
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.044
--- 50000 row(s) unloaded.
>>log;
@@ -517,13 +541,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:39:48.767
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:39:48.767
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.740
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.137
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:39:48.919
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.151
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:39:48.919
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:39:48.943
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.024
--- 20000 row(s) unloaded.
>>log;
@@ -541,13 +568,16 @@ cat /tmp/merged_customer_demogs | wc -l
+>select * from trafodion.hbase.customer_demographics
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.011
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:39:48.943
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:39:51.647
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:39:51.647
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:03.066
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.062
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:39:51.777
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.130
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:39:51.777
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:39:51.811
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.033
--- 20000 row(s) unloaded.
>>log;
@@ -578,13 +608,16 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:39:51.811
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:39:55.327
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:39:55.327
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.312
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.042
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:39:55.715
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.388
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:39:55.715
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:39:55.753
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.038
--- 20000 row(s) unloaded.
>>
@@ -601,13 +634,16 @@ regrhadoop.ksh fs -du -s /bulkload/customer_demographics_salt/merged_customer_de
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:39:55.753
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:39:58.168
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2016-12-11 21:39:58.168
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.581
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.053
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:39:58.350
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.182
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:39:58.350
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:39:58.382
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.032
--- 20000 row(s) unloaded.
>>
@@ -639,11 +675,13 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:00.958
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.006
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:00.958
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.466
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:01.143
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.185
--- 20000 row(s) unloaded.
>>
@@ -662,13 +700,16 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/file* | grep file |
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.012
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:40:01.143
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:05.852
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.010
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:05.852
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.457
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.060
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:05.999
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.147
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:40:05.999
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:40:06.456
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.046
--- 20000 row(s) unloaded.
>>
@@ -800,13 +841,16 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.007
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:13.777
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:13.777
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.610
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.044
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:13.908
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.131
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:40:13.908
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:40:13.948
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.040
--- 20000 row(s) unloaded.
>>log;
@@ -840,13 +884,16 @@ regrhadoop.ksh fs -ls /bulkload/customer_demographics_salt/merged* | grep merge
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.009
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:16.517
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:16.517
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.582
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.091
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:16.665
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.148
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:40:16.665
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:40:16.708
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.043
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -895,11 +942,13 @@ CD_DEMO_SK CD_GENDER
+>select * from trafodion.hbase.customer_demographics_salt
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:17.956
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:17.956
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.547
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:18.734
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.117
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -907,7 +956,7 @@ Task: EXTRACT Status: Ended ET: 00:00:01.547
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1476865840, failedModTS = 1476865842, failedLoc = hdfs://localhost:30200/user/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1481492416, failedModTS = 1481492417, failedLoc = hdfs://localhost:27000/user/hive/exttables/unload_customer_demographics
(EXPR)
--------------------
@@ -949,11 +998,13 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/hive/exttables/unload_customer_address'
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:19.837
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:19.837
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:03.943
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:20.118
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.281
--- 50000 row(s) unloaded.
>>--sh sleep 10;
@@ -1002,11 +1053,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/hive/exttables/unload_customer_address'
+>select * from trafodion.hbase.customer_address ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.006
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:22.547
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:22.547
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:00.996
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:22.861
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.315
--- 50000 row(s) unloaded.
>>--sh sleep 10;
@@ -1014,7 +1067,7 @@ Task: EXTRACT Status: Ended ET: 00:00:00.996
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1476865850, failedModTS = 1476865853, failedLoc = hdfs://localhost:30200/user/hive/exttables/unload_customer_address
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1481492420, failedModTS = 1481492422, failedLoc = hdfs://localhost:27000/user/hive/exttables/unload_customer_address
(EXPR)
--------------------
@@ -1068,11 +1121,13 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/hive/exttables/unload_customer'
+>select * from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:26.566
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:26.566
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:03.895
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:27.991
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.425
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -1122,11 +1177,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>INTO '/user/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:29.751
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:29.751
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.292
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:29.896
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.145
--- 20000 row(s) unloaded.
>>--sh sleep 10;
@@ -1175,13 +1232,16 @@ CD_DEMO_SK CD_GENDER
+>INTO '/bulkload/customer_address'
+>select * from trafodion.hbase.customer_address where ca_address_sk < 100;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:31.137
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:31.137
Rows Processed: 99
-Task: EXTRACT Status: Ended ET: 00:00:01.275
-Task: MERGE FILES Status: Started
-Task: MERGE FILES Status: Ended ET: 00:00:00.029
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:31.178
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.041
+Task: MERGE FILES Status: Started Time: 2016-12-11 21:40:31.178
+Task: MERGE FILES Status: Ended Time: 2016-12-11 21:40:31.211
+Task: MERGE FILES Status: Ended Elapsed Time: 00:00:00.032
--- 99 row(s) unloaded.
>>
@@ -1216,11 +1276,13 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/*
+>INTO '/user/hive/exttables/unload_store_sales_summary'
+>select ss_sold_date_sk,ss_store_sk, sum (ss_quantity) from store_sales_salt group by ss_sold_date_sk ,ss_store_sk;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:40:31.211
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:34.107
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:34.107
Rows Processed: 12349
-Task: EXTRACT Status: Ended ET: 00:00:05.114
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:36.659
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:02.552
--- 12349 row(s) unloaded.
>>--sh sleep 10;
@@ -1338,11 +1400,13 @@ SS_SOLD_DATE_SK SS_STORE_SK SS_QUANTITY
+>INTO '/user/hive/exttables/unload_customer_and_address'
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:38.262
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:38.262
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:05.860
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:39.469
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:01.443
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -1389,11 +1453,13 @@ C_CUSTOMER_SK C_CUSTOMER_ID
+>INTO '/user/hive/exttables/unload_customer_address'
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.016
-Task: EXTRACT Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:40:42.118
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: EXTRACT Status: Started Time: 2016-12-11 21:40:42.118
Rows Processed: 1998
-Task: EXTRACT Status: Ended ET: 00:00:00.407
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:40:42.243
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.126
--- 1998 row(s) unloaded.
>>--sh sleep 10;
@@ -1504,7 +1570,7 @@ ESP_EXCHANGE ============================== SEQ_NO 3 ONLY CHILD 2
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT_SNAP111
- snapshot_temp_location /bulkload/20161019083132/
+ snapshot_temp_location /bulkload/20161211214044/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1588,7 +1654,7 @@ grep -i -e 'explain snp' -e snapshot -e full_table_name -e esp_exchange LOG018_S
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20161019083142/
+ snapshot_temp_location /bulkload/20161211214049/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1683,12 +1749,12 @@ ESP_EXCHANGE ============================== SEQ_NO 6 ONLY CHILD 5
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_SALT
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_SALT_SNAP111
- snapshot_temp_location /bulkload/20161019083206/
+ snapshot_temp_location /bulkload/20161211214105/
ESP_EXCHANGE ============================== SEQ_NO 2 ONLY CHILD 1
use_snapshot_scan ...... TRUE
full_table_name ........ TRAFODION.HBASE.CUSTOMER_ADDRESS
snapshot_name .......... TRAFODION.HBASE.CUSTOMER_ADDRESS_SNAP111
- snapshot_temp_location /bulkload/20161019083206/
+ snapshot_temp_location /bulkload/20161211214105/
grep -i -e 'explain reg' -e snapshot -e full_table_name -e esp_exchange LOG018_REGULAR_SCAN_PLAN.TXT | grep -v snapshot_scan_run_id
>>--no snapshot
>>explain reg;
@@ -1806,14 +1872,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>select * from customer_address
+><<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: VERIFY SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:19.396
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: VERIFY SNAPSHO Status: Started Time: 2016-12-11 21:42:19.396
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.735
-Task: EXTRACT Status: Started
+Task: VERIFY SNAPSHO Status: Ended Time: 2016-12-11 21:42:19.648
+Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.253
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:19.648
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:02.488
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:20.138
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.489
--- 50000 row(s) unloaded.
>>
@@ -1884,14 +1953,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>INTO '/user/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.002
-Task: VERIFY SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:22.305
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.002
+Task: VERIFY SNAPSHO Status: Started Time: 2016-12-11 21:42:22.305
Snapshots verified: 1
-Task: VERIFY SNAPSHO Status: Ended ET: 00:00:00.379
-Task: EXTRACT Status: Started
+Task: VERIFY SNAPSHO Status: Ended Time: 2016-12-11 21:42:22.524
+Task: VERIFY SNAPSHO Status: Ended Elapsed Time: 00:00:00.219
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:22.524
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:22.448
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:29.407
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:06.883
--- 20000 row(s) unloaded.
>>
@@ -1938,17 +2010,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.008
-Task: CREATE SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:30.660
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.007
+Task: CREATE SNAPSHO Status: Started Time: 2016-12-11 21:42:30.660
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.473
-Task: EXTRACT Status: Started
+Task: CREATE SNAPSHO Status: Ended Time: 2016-12-11 21:42:30.986
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.326
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:30.986
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.990
-Task: DELETE SNAPSHO Status: Started
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:31.840
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.854
+Task: DELETE SNAPSHO Status: Started Time: 2016-12-11 21:42:31.840
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.039
+Task: DELETE SNAPSHO Status: Ended Time: 2016-12-11 21:42:31.852
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.012
--- 20000 row(s) unloaded.
>>
@@ -1956,7 +2032,7 @@ Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.039
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1476866073, failedModTS = 1476866079, failedLoc = hdfs://localhost:30200/user/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1481492549, failedModTS = 1481492550, failedLoc = hdfs://localhost:27000/user/hive/exttables/unload_customer_demographics
(EXPR)
--------------------
@@ -1999,17 +2075,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/hive/exttables/unload_customer_demographics'
+>select * from trafodion.hbase.customer_demographics_salt <<+ cardinality 10e10 >>;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.012
-Task: CREATE SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:33.652
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.009
+Task: CREATE SNAPSHO Status: Started Time: 2016-12-11 21:42:33.652
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.763
-Task: EXTRACT Status: Started
+Task: CREATE SNAPSHO Status: Ended Time: 2016-12-11 21:42:34.164
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.513
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:34.164
Rows Processed: 20000
-Task: EXTRACT Status: Ended ET: 00:00:01.701
-Task: DELETE SNAPSHO Status: Started
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:34.982
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.817
+Task: DELETE SNAPSHO Status: Started Time: 2016-12-11 21:42:34.982
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005
+Task: DELETE SNAPSHO Status: Ended Time: 2016-12-11 21:42:34.987
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.006
--- 20000 row(s) unloaded.
>>
@@ -2025,7 +2105,7 @@ Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005
*** WARNING[8597] Statement was automatically retried 1 time(s). Delay before each retry was 0 seconds. See next entry for the error that caused this retry.
-*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1476866080, failedModTS = 1476866084, failedLoc = hdfs://localhost:30200/user/hive/exttables/unload_customer_demographics
+*** WARNING[8436] Mismatch detected between compiletime and runtime hive table definitions. DataModMismatchDetails: compiledModTS = 1481492551, failedModTS = 1481492554, failedLoc = hdfs://localhost:27000/user/hive/exttables/unload_customer_demographics
CD_DEMO_SK CD_GENDER CD_MARITAL_STATUS CD_EDUCATION_STATUS CD_PURCHASE_ESTIMATE CD_CREDIT_RATING CD_DEP_COUNT CD_DEP_EMPLOYED_COUNT CD_DEP_COLLEGE_COUNT
----------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- ---------------------------------------------------------------------------------------------------- -------------------- ---------------------------------------------------------------------------------------------------- ------------ --------------------- --------------------
@@ -2061,17 +2141,21 @@ CD_DEMO_SK CD_GENDER
+>INTO '/user/hive/exttables/unload_customer_address'
+>select * from customer_address where ca_address_sk < 1000 union select * from customer_address where ca_address_sk > 40000 and ca_address_sk < 41000;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.004
-Task: CREATE SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:36.735
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.003
+Task: CREATE SNAPSHO Status: Started Time: 2016-12-11 21:42:36.735
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:00.873
-Task: EXTRACT Status: Started
+Task: CREATE SNAPSHO Status: Ended Time: 2016-12-11 21:42:37.610
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.325
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:37.610
Rows Processed: 1998
-Task: EXTRACT Status: Ended ET: 00:00:00.824
-Task: DELETE SNAPSHO Status: Started
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:37.337
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.276
+Task: DELETE SNAPSHO Status: Started Time: 2016-12-11 21:42:37.337
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.005
+Task: DELETE SNAPSHO Status: Ended Time: 2016-12-11 21:42:37.343
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.006
--- 1998 row(s) unloaded.
>>
@@ -2145,17 +2229,21 @@ CA_ADDRESS_SK CA_ADDRESS_ID
+>INTO '/user/hive/exttables/unload_customer_and_address'
+>select * from trafodion.hbase.customer_salt c join trafodion.hbase.customer_address ca on c.c_current_addr_sk = ca.ca_address_sk ;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.005
-Task: CREATE SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: -001-23--2 00:00:00.000
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:38.777
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.005
+Task: CREATE SNAPSHO Status: Started Time: 2016-12-11 21:42:38.777
Snapshots created: 2
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:02.912
-Task: EXTRACT Status: Started
+Task: CREATE SNAPSHO Status: Ended Time: 2016-12-11 21:42:40.430
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:01.653
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:40.430
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:04.850
-Task: DELETE SNAPSHO Status: Started
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:42.706
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:02.277
+Task: DELETE SNAPSHO Status: Started Time: 2016-12-11 21:42:42.706
Snapshots deleted: 2
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.019
+Task: DELETE SNAPSHO Status: Ended Time: 2016-12-11 21:42:42.718
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.012
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -2222,17 +2310,21 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
+>INTO '/user/hive/exttables/unload_customer_name'
+>select c_first_name,c_last_name from trafodion.hbase.customer_salt;
Task: UNLOAD Status: Started
-Task: EMPTY TARGET Status: Started
-Task: EMPTY TARGET Status: Ended ET: 00:00:00.010
-Task: CREATE SNAPSHO Status: Started
+Task: EMPTY TARGET Status: Started Time: 2016-12-11 21:42:42.718
+Task: EMPTY TARGET Status: Ended Time: 2016-12-11 21:42:44.587
+Task: EMPTY TARGET Status: Ended Elapsed Time: 00:00:00.004
+Task: CREATE SNAPSHO Status: Started Time: 2016-12-11 21:42:44.587
Snapshots created: 1
-Task: CREATE SNAPSHO Status: Ended ET: 00:00:01.126
-Task: EXTRACT Status: Started
+Task: CREATE SNAPSHO Status: Ended Time: 2016-12-11 21:42:44.908
+Task: CREATE SNAPSHO Status: Ended Elapsed Time: 00:00:00.321
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:44.908
Rows Processed: 100000
-Task: EXTRACT Status: Ended ET: 00:00:01.713
-Task: DELETE SNAPSHO Status: Started
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:45.546
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.638
+Task: DELETE SNAPSHO Status: Started Time: 2016-12-11 21:42:45.546
Snapshots deleted: 1
-Task: DELETE SNAPSHO Status: Ended ET: 00:00:00.006
+Task: DELETE SNAPSHO Status: Ended Time: 2016-12-11 21:42:45.552
+Task: DELETE SNAPSHO Status: Ended Elapsed Time: 00:00:00.006
--- 100000 row(s) unloaded.
>>--sh sleep 10;
@@ -2276,16 +2368,16 @@ C_FIRST_NAME
>>--unload 100 --should give error [8447]
>>unload into '//\a//c' select * from CUSTOMER_ADDRESS;
Task: UNLOAD Status: Started
-Task: EXTRACT Status: Started
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:45.552
*** ERROR[8447] An error occurred during hdfs access. Error Detail: Java exception in hdfsCreate(). java.io.IOException: No FileSystem for scheme: null
-org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2584)
-org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2591)
-org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:91)
-org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2630)
-org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2612)
-org.apache.hadoop.fs.FileSystem.get(FileSystem.java:370)
-org.trafodion.sql.SequenceFileWriter.hdfsCreate(SequenceFileWriter.java:163)
+org.apache.hadoop.fs.FileSystem.getFileSystemClass(FileSystem.java:2676)
+org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2690)
+org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:94)
+org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2733)
+org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2715)
+org.apache.hadoop.fs.FileSystem.get(FileSystem.java:382)
+org.trafodion.sql.SequenceFileWriter.hdfsCreate(SequenceFileWriter.java:155)
--- 0 row(s) unloaded.
>>
@@ -2310,9 +2402,10 @@ unload with delimiter 0 into '/bulkload/test' select * from CUSTOMER_ADDRESS;
>>--unload 103 -- should not give an error
>>unload with delimiter '\a' into '/bulkload/test' select * from customer_address;
Task: UNLOAD Status: Started
-Task: EXTRACT Status: Started
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:45.552
Rows Processed: 50000
-Task: EXTRACT Status: Ended ET: 00:00:00.885
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:47.906
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.303
--- 50000 row(s) unloaded.
>>--unload 24 -- should give an error
@@ -2375,9 +2468,10 @@ regrhadoop.ksh fs -rm /user/hive/exttables/unload_customer_demographics/*
+>INTO '/user/hive/exttables/unload_customer_demographics'
+>(select * from trafodion.hbase.customer_demographics_salt) ;
Task: UNLOAD Status: Started
-Task: EXTRACT Status: Started
+Task: EXTRACT Status: Started Time: 2016-12-11 21:42:47.906
Rows Processed but NOT Written to Disk: 20000
-Task: EXTRACT Status: Ended ET: 00:00:00.291
+Task: EXTRACT Status: Ended Time: 2016-12-11 21:42:50.516
+Task: EXTRACT Status: Ended Elapsed Time: 00:00:00.134
--- 20000 row(s) unloaded.
>>--sh sleep 10;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/hive/FILTER015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/FILTER015 b/core/sql/regress/hive/FILTER015
index d0e9725..e7d6168 100755
--- a/core/sql/regress/hive/FILTER015
+++ b/core/sql/regress/hive/FILTER015
@@ -28,7 +28,8 @@ if [ "$fil" = "" ]; then
fi
sed "
-s/ET:.*/ET:xxxxxxxxxx/g
+s/Logging Location:.*/Loggin Location:@loggingLocation@/
+s/Time:.*/Time:@time@/
s/hash2 partitioned.*ways on/hash2 partitioned N ways on/g
s/\(esp_exchange.*\)[1-9](hash2):[1-9]/\1 N(hash2):M/g
s/\(esp_exchange.*\)1:[1-9](hash2)/\1 1:N(hash2)/g
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/hive/FILTER018
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/FILTER018 b/core/sql/regress/hive/FILTER018
index fff0757..daf8019 100755
--- a/core/sql/regress/hive/FILTER018
+++ b/core/sql/regress/hive/FILTER018
@@ -28,7 +28,8 @@ if [ "$fil" = "" ]; then
fi
sed "
-s/ET:.*/ET:xxxxxxxxxx/g
+s/Logging Location:.*/Loggin Location:@loggingLocation@/
+s/Time:.*/Time:@time@/
s/snapshot_temp_location.*/snapshot_temp_location location/g
s/\(778224\).*\(778224\)\(.*\)/\1\3/g
s/\(78431\).*\(78431\)\(.*\)/\1\3/g
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index 149f097..d658175 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -3330,7 +3330,7 @@ XDDkwd__(SUBQUERY_UNNESTING, "ON"),
DDkwd__(TRAF_LOAD_CONTINUE_ON_ERROR, "OFF"),
DD_____(TRAF_LOAD_ERROR_COUNT_ID, "" ),
DD_____(TRAF_LOAD_ERROR_COUNT_TABLE, "ERRORCOUNTER" ),
- DD_____(TRAF_LOAD_ERROR_LOGGING_LOCATION, "/bulkload/logs/" ),
+ DD_____(TRAF_LOAD_ERROR_LOGGING_LOCATION, "/bulkload/logs" ),
DDint__(TRAF_LOAD_FLUSH_SIZE_IN_KB, "1024"),
DDkwd__(TRAF_LOAD_FORCE_CIF, "ON"),
DDkwd__(TRAF_LOAD_LOG_ERROR_ROWS, "OFF"),
[7/8] incubator-trafodion git commit: [TRAFODION-2351] Bulk load with
log error rows enhancements
Posted by se...@apache.org.
[TRAFODION-2351] Bulk load with log error rows enhancements
Fixes to take care of review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/884eee0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/884eee0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/884eee0c
Branch: refs/heads/master
Commit: 884eee0c622aedd9dfecceb8be75ada32a182d2a
Parents: e0244dd
Author: selvaganesang <se...@esgyn.com>
Authored: Mon Dec 19 05:47:05 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Mon Dec 19 05:47:05 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtilCommon.cpp | 1 -
core/sql/executor/ExHbaseAccess.cpp | 6 +++++-
core/sql/executor/ExHbaseAccess.h | 5 +++--
core/sql/executor/ExHbaseIUD.cpp | 7 ++++++-
core/sql/executor/ExHdfsScan.cpp | 7 ++++++-
core/sql/executor/ExHdfsScan.h | 2 +-
6 files changed, 21 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExExeUtilCommon.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilCommon.cpp b/core/sql/executor/ExExeUtilCommon.cpp
index a92e786..f400ba0 100644
--- a/core/sql/executor/ExExeUtilCommon.cpp
+++ b/core/sql/executor/ExExeUtilCommon.cpp
@@ -648,7 +648,6 @@ short ExExeUtilTcb::executeQuery(char * task,
if (displayEndTime)
{
- char timeBuf[200];
getTimestampAsString(endTime_, timeBuf);
getStatusString(task, "Ended", object, buf, FALSE, timeBuf);
if (moveRowToUpQueue(buf, 0, &rc))
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp
index d56d9df..b102aef 100644
--- a/core/sql/executor/ExHbaseAccess.cpp
+++ b/core/sql/executor/ExHbaseAccess.cpp
@@ -3033,13 +3033,17 @@ void ExHbaseAccessTcb::setRowID(char *rowId, Lng32 rowIdLen)
memcpy(rowID_.val, rowId, rowIdLen);
}
-void ExHbaseAccessTcb::buildLoggingFileName(
+void ExHbaseAccessTcb::buildLoggingFileName(NAHeap *heap,
const char * currCmdLoggingLocation,
const char *tableName,
const char * loggingFileNamePrefix,
Lng32 instId,
char * loggingFileName)
{
+ if (loggingFileName != NULL)
+ NADELETEBASIC(loggingFileName, heap);
+ short logLen = strlen(currCmdLoggingLocation)+strlen(loggingFileNamePrefix)+strlen(tableName)+100;
+ loggingFileName = new (heap) char[logLen];
sprintf(loggingFileName, "%s/%s_%s_%d",
currCmdLoggingLocation, loggingFileNamePrefix, tableName, instId);
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index a1e25c2..d2f5a7e 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -185,7 +185,8 @@ public:
char *logId,
const char *tableName,
char * currCmdLoggingLocation);
- static void buildLoggingFileName(const char * currCmdLoggingLocation,
+ static void buildLoggingFileName(NAHeap *heap,
+ const char * currCmdLoggingLocation,
const char *tableName,
const char * loggingFileNamePrefix,
Lng32 instId,
@@ -943,7 +944,7 @@ class ExHbaseAccessBulkLoadPrepSQTcb: public ExHbaseAccessUpsertVsbbSQTcb
Text importLocation_;
Text hFileName_;
- char loggingFileName_[1000];
+ char *loggingFileName_;
NABoolean LoggingFileCreated_ ;
ComCondition * lastErrorCnd_;
std::vector<UInt32> posVec_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp
index 6b80f3f..b89a3f2 100644
--- a/core/sql/executor/ExHbaseIUD.cpp
+++ b/core/sql/executor/ExHbaseIUD.cpp
@@ -1067,6 +1067,7 @@ ExHbaseAccessBulkLoadPrepSQTcb::ExHbaseAccessBulkLoadPrepSQTcb(
prevRowId_ (NULL),
hdfs_(NULL),
hdfsSampleFile_(NULL),
+ loggingFileName_(NULL),
lastErrorCnd_(NULL)
{
hFileParamsInitialized_ = false;
@@ -1076,7 +1077,7 @@ ExHbaseAccessBulkLoadPrepSQTcb::ExHbaseAccessBulkLoadPrepSQTcb(
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
- ExHbaseAccessTcb::buildLoggingFileName(((ExHbaseAccessTdb &)hbaseAccessTdb).getLoggingLocation(),
+ ExHbaseAccessTcb::buildLoggingFileName((NAHeap *)getHeap(), ((ExHbaseAccessTdb &)hbaseAccessTdb).getLoggingLocation(),
// (char *)((ExHbaseAccessTdb &)hbaseAccessTdb).getErrCountRowId(),
((ExHbaseAccessTdb &)hbaseAccessTdb).getTableName(),
"traf_upsert_err",
@@ -1088,6 +1089,10 @@ ExHbaseAccessBulkLoadPrepSQTcb::ExHbaseAccessBulkLoadPrepSQTcb(
ExHbaseAccessBulkLoadPrepSQTcb::~ExHbaseAccessBulkLoadPrepSQTcb()
{
+ if (loggingFileName_ != NULL) {
+ NADELETEBASIC(loggingFileName_, getHeap());
+ loggingFileName_ = NULL;
+ }
// Flush and close sample file if used
if (hdfs_)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 9ce17d5..d80f882 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -119,6 +119,7 @@ ExHdfsScanTcb::ExHdfsScanTcb(
, loggingErrorDiags_(NULL)
, runTimeRanges_(NULL)
, numRunTimeRanges_(0)
+ , loggingFileName_(NULL)
{
Space * space = (glob ? glob->getSpace() : 0);
CollHeap * heap = (glob ? glob->getDefaultHeap() : 0);
@@ -187,7 +188,7 @@ ExHdfsScanTcb::ExHdfsScanTcb(
registerResizeSubtasks();
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
- ExHbaseAccessTcb::buildLoggingFileName(((ExHdfsScanTdb &)hdfsScanTdb).getLoggingLocation(),
+ ExHbaseAccessTcb::buildLoggingFileName((NAHeap *)getHeap(), ((ExHdfsScanTdb &)hdfsScanTdb).getLoggingLocation(),
// (char *)((ExHdfsScanTdb &)hdfsScanTdb).getErrCountRowId(),
((ExHdfsScanTdb &)hdfsScanTdb).tableName(),
"hive_scan_err",
@@ -214,6 +215,10 @@ ExHdfsScanTcb::~ExHdfsScanTcb()
void ExHdfsScanTcb::freeResources()
{
+ if (loggingFileName_ != NULL) {
+ NADELETEBASIC(loggingFileName_, getHeap());
+ loggingFileName_ = NULL;
+ }
if (workAtp_)
{
workAtp_->release();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/884eee0c/core/sql/executor/ExHdfsScan.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.h b/core/sql/executor/ExHdfsScan.h
index 2df1ae0..25f3ac3 100644
--- a/core/sql/executor/ExHdfsScan.h
+++ b/core/sql/executor/ExHdfsScan.h
@@ -285,7 +285,7 @@ protected:
bool seqScanAgain_;
char cursorId_[8];
- char loggingFileName_[1000];
+ char *loggingFileName_;
NABoolean LoggingFileCreated_ ;
char * hdfsLoggingRow_;
char * hdfsLoggingRowEnd_;
[3/8] incubator-trafodion git commit: [TRAFODION-2356] HBase snapshot
concept needs to be contained within HBaseClient in Trafodion
Posted by se...@apache.org.
[TRAFODION-2356] HBase snapshot concept needs to be contained within
HBaseClient in Trafodion
Moved snapshot creation/deletion from SequenceFileWriter to HBaseClient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1c356227
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1c356227
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1c356227
Branch: refs/heads/master
Commit: 1c356227da4706b07962350eea6cfb9c9ad5304c
Parents: e0dab3e
Author: selvaganesang <se...@esgyn.com>
Authored: Tue Nov 22 04:20:29 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Tue Nov 22 04:20:29 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 1 +
core/sql/executor/ExExeUtilLoad.cpp | 45 +++++--
core/sql/executor/ExHbaseAccess.cpp | 30 +++++
core/sql/executor/ExHbaseAccess.h | 1 +
core/sql/executor/HBaseClient_JNI.cpp | 129 +++++++++++++++++--
core/sql/executor/HBaseClient_JNI.h | 12 ++
core/sql/executor/SequenceFileReader.cpp | 127 ------------------
core/sql/executor/SequenceFileReader.h | 12 --
core/sql/exp/ExpHbaseDefs.h | 6 +
core/sql/exp/ExpHbaseInterface.cpp | 45 +++++++
core/sql/exp/ExpHbaseInterface.h | 12 +-
.../java/org/trafodion/sql/HBaseClient.java | 41 ++++++
.../java/org/trafodion/sql/HBulkLoadClient.java | 2 -
.../org/trafodion/sql/SequenceFileWriter.java | 70 +---------
14 files changed, 307 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index f3fccb8..73bee38 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -3678,6 +3678,7 @@ class ExExeUtilHBaseBulkUnLoadTcb : public ExExeUtilTcb
NAList<struct snapshotStruct *> * snapshotsList_;
NABoolean emptyTarget_;
NABoolean oneFile_;
+ ExpHbaseInterface * ehi_;
};
class ExExeUtilHbaseUnLoadPrivateState : public ex_tcb_private_state
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index d11f182..73e0d2f 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -66,6 +66,8 @@ using std::ofstream;
#include "ExpLOBexternal.h"
#include "str.h"
#include "ExpHbaseInterface.h"
+#include "ExHbaseAccess.h"
+
///////////////////////////////////////////////////////////////////
ex_tcb * ExExeUtilCreateTableAsTdb::build(ex_globals * glob)
{
@@ -1884,6 +1886,13 @@ ExExeUtilHBaseBulkUnLoadTcb::ExExeUtilHBaseBulkUnLoadTcb(
oneFile_(FALSE)
{
sequenceFileWriter_ = NULL;
+ int jniDebugPort = 0;
+ int jniDebugTimeout = 0;
+ ehi_ = ExpHbaseInterface::newInstance(getGlobals()->getDefaultHeap(),
+ (char*)"", //Later may need to change to hblTdb.server_,
+ (char*)"", //Later may need to change to hblTdb.zkPort_,
+ jniDebugPort,
+ jniDebugTimeout);
qparent_.down->allocatePstate(this);
}
@@ -1911,9 +1920,10 @@ void ExExeUtilHBaseBulkUnLoadTcb::freeResources()
NADELETE(sequenceFileWriter_, SequenceFileWriter, getMyHeap());
sequenceFileWriter_ = NULL;
}
-
-
+ NADELETE(ehi_, ExpHbaseInterface, getGlobals()->getDefaultHeap());
+ ehi_ = NULL;
}
+
ExExeUtilHBaseBulkUnLoadTcb::~ExExeUtilHBaseBulkUnLoadTcb()
{
freeResources();
@@ -2058,7 +2068,7 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
short retcode = 0;
short rc;
SFW_RetCode sfwRetCode = SFW_OK;
-
+ Lng32 hbcRetCode = HBC_OK;
// if no parent request, return
if (qparent_.down->isEmpty())
return WORK_OK;
@@ -2103,6 +2113,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
break;
}
}
+ if ((retcode = ehi_->init(NULL)) != HBASE_ACCESS_SUCCESS)
+ {
+ handleError();
+ step_ = UNLOAD_END_ERROR_;
+ break;
+ }
if (!hblTdb().getOverwriteMergeFile() && hblTdb().getMergePath() != NULL)
{
NABoolean exists = FALSE;
@@ -2234,12 +2250,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
for ( int i = 0 ; i < snapshotsList_->entries(); i++)
{
if (createSnp)
- sfwRetCode = sequenceFileWriter_->createSnapshot( *snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName);
+ hbcRetCode = ehi_->createSnapshot( *snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName);
else
{
NABoolean exist = FALSE;
- sfwRetCode = sequenceFileWriter_->verifySnapshot(*snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName, exist);
- if ( sfwRetCode == SFW_OK && !exist)
+ hbcRetCode = ehi_->verifySnapshot(*snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName, exist);
+ if ( hbcRetCode == HBC_OK && !exist)
{
ComDiagsArea * da = getDiagsArea();
*da << DgSqlCode(-8112)
@@ -2249,9 +2265,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
break;
}
}
- if (sfwRetCode != SFW_OK)
+ if (hbcRetCode != HBC_OK)
{
- createHdfsFileError(sfwRetCode);
+ ExHbaseAccessTcb::setupError((NAHeap *)getMyHeap(),qparent_, hbcRetCode,
+ "HBaseClient_JNI::createSnapshot/verifySnapshot",
+ snapshotsList_->at(i)->snapshotName->data() );
+ handleError();
step_ = UNLOAD_END_ERROR_;
break;
}
@@ -2307,10 +2326,13 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
return rc;
for ( int i = 0 ; i < snapshotsList_->entries(); i++)
{
- sfwRetCode = sequenceFileWriter_->deleteSnapshot( *snapshotsList_->at(i)->snapshotName);
- if (sfwRetCode != SFW_OK)
+ hbcRetCode = ehi_->deleteSnapshot( *snapshotsList_->at(i)->snapshotName);
+ if (hbcRetCode != HBC_OK)
{
- createHdfsFileError(sfwRetCode);
+ ExHbaseAccessTcb::setupError((NAHeap *)getMyHeap(),qparent_, hbcRetCode,
+ "HBaseClient_JNI::createSnapshot/verifySnapshot",
+ snapshotsList_->at(i)->snapshotName->data() );
+ handleError();
step_ = UNLOAD_END_ERROR_;
break;
}
@@ -2351,6 +2373,7 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
case UNLOAD_END_:
case UNLOAD_END_ERROR_:
{
+ ehi_->close();
if (restoreCQD("TRAF_TABLE_SNAPSHOT_SCAN") < 0)
{
step_ = UNLOAD_ERROR_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp
index 53a4dd1..a7b7f4e 100644
--- a/core/sql/executor/ExHbaseAccess.cpp
+++ b/core/sql/executor/ExHbaseAccess.cpp
@@ -686,6 +686,36 @@ short ExHbaseAccessTcb::moveRowToUpQueue(const char * row, Lng32 len,
return 0;
}
+short ExHbaseAccessTcb::setupError(NAHeap *heap, ex_queue_pair &qparent, Lng32 retcode, const char * str, const char * str2)
+{
+ ContextCli *currContext = GetCliGlobals()->currContext();
+ // Make sure retcode is positive.
+ if (retcode < 0)
+ retcode = -retcode;
+
+ if ((ABS(retcode) >= HBASE_MIN_ERROR_NUM) &&
+ (ABS(retcode) <= HBASE_MAX_ERROR_NUM))
+ {
+ ex_queue_entry *pentry_down = qparent.down->getHeadEntry();
+
+ Lng32 cliError = 0;
+
+ Lng32 intParam1 = -retcode;
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(heap, &diagsArea,
+ (ExeErrorCode)(8448), NULL, &intParam1,
+ &cliError, NULL,
+ (str ? (char*)str : (char*)" "),
+ getHbaseErrStr(retcode),
+ (str2 ? (char*)str2 :
+ (char *)currContext->getJniErrorStr().data()));
+ pentry_down->setDiagsArea(diagsArea);
+ return -1;
+ }
+
+ return 0;
+}
+
short ExHbaseAccessTcb::setupError(Lng32 retcode, const char * str, const char * str2)
{
ContextCli *currContext = GetCliGlobals()->currContext();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index eb0792c..03cb052 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -187,6 +187,7 @@ public:
const char * loggingFileNamePrefix,
Lng32 instId,
char * loggingFileName);
+ static short setupError(NAHeap *heap, ex_queue_pair &qparent, Lng32 retcode, const char * str, const char * str2 = NULL);
protected:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/HBaseClient_JNI.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp
index 272ee5b..603c42e 100644
--- a/core/sql/executor/HBaseClient_JNI.cpp
+++ b/core/sql/executor/HBaseClient_JNI.cpp
@@ -99,6 +99,12 @@ static const char* const hbcErrorEnumStr[] =
,"Preparing parameters for getKeys()."
,"Preparing parameters for listAll()."
,"Preparing parameters for getRegionStats()."
+ ,"JNI NewStringUTF() in createSnapshot()."
+ ,"Java exception in createSnapshot()."
+ ,"JNI NewStringUTF() in deleteSnapshot()."
+ ,"Java exception in deleteSnapshot()."
+ ,"JNI NewStringUTF() in verifySnapshot()."
+ ,"Java exception in verifySnapshot()."
};
//////////////////////////////////////////////////////////////////////////////
@@ -281,6 +287,12 @@ HBC_RetCode HBaseClient_JNI::init()
JavaMethods_[JM_HBC_GETSTARTKEYS ].jm_signature = "(Ljava/lang/String;Z)[[B";
JavaMethods_[JM_HBC_GETENDKEYS ].jm_name = "getEndKeys";
JavaMethods_[JM_HBC_GETENDKEYS ].jm_signature = "(Ljava/lang/String;Z)[[B";
+ JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_name = "createSnapshot";
+ JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
+ JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_name = "deleteSnapshot";
+ JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_signature = "(Ljava/lang/String;)Z";
+ JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_name = "verifySnapshot";
+ JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
rc = (HBC_RetCode)JavaObjectInterface::init(className, javaClass_, JavaMethods_, (Int32)JM_LAST, javaMethodsInitialized_);
javaMethodsInitialized_ = TRUE;
pthread_mutex_unlock(&javaMethodsInitMutex_);
@@ -1814,10 +1826,8 @@ HVC_RetCode HiveClient_JNI::hdfsCreateFile(const char* path)
{
QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HiveClient_JNI::hdfsCreate(%s) called.", path);
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return HVC_ERROR_HDFS_CREATE_EXCEPTION;
- }
+ if (initJNIEnv() != JOI_OK)
+ return HVC_ERROR_INIT_PARAM;
jstring js_path = jenv_->NewStringUTF(path);
if (js_path == NULL) {
@@ -1852,10 +1862,8 @@ HVC_RetCode HiveClient_JNI::hdfsCreateFile(const char* path)
{
QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HiveClient_JNI::hdfsWrite(%ld) called.", len);
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return HVC_ERROR_HDFS_WRITE_EXCEPTION;
- }
+ if (initJNIEnv() != JOI_OK)
+ return HVC_ERROR_INIT_PARAM;
//Write the requisite bytes into the file
jbyteArray jbArray = jenv_->NewByteArray( len);
@@ -3027,7 +3035,112 @@ NAArray<HbaseStr>* HBaseClient_JNI::getKeys(Int32 funcIndex, NAHeap *heap, const
return retArray;
}
+HBC_RetCode HBaseClient_JNI::createSnapshot( const NAString& tableName, const NAString& snapshotName)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::createSnapshot(%s, %s) called.",
+ tableName.data(), snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_tableName = jenv_->NewStringUTF(tableName.data());
+ if (js_tableName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_CREATE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_PARAM;
+ }
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_CREATE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_CREATE_SNAPSHOT].methodID, js_tableName, js_snapshotName);
+
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::createSnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_EXCEPTION;
+ }
+
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
+
+HBC_RetCode HBaseClient_JNI::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::verifySnapshot(%s, %s) called.",
+ tableName.data(), snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_tableName = jenv_->NewStringUTF(tableName.data());
+ if (js_tableName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_VERIFY_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_PARAM;
+ }
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_VERIFY_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].methodID, js_tableName, js_snapshotName);
+
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::verifySnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
+ }
+
+ exist = jresult;
+
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
+
+HBC_RetCode HBaseClient_JNI::deleteSnapshot( const NAString& snapshotName)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::deleteSnapshot(%s) called.",
+ snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_DELETE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_DELETE_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DELETE_SNAPSHOT].methodID, js_snapshotName);
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::deleteSnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_DELETE_SNAPSHOT_EXCEPTION;
+ }
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
// ===========================================================================
// ===== Class HTableClient
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/HBaseClient_JNI.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.h b/core/sql/executor/HBaseClient_JNI.h
index d0eeeb2..08464ab 100644
--- a/core/sql/executor/HBaseClient_JNI.h
+++ b/core/sql/executor/HBaseClient_JNI.h
@@ -398,6 +398,12 @@ typedef enum {
,HBC_ERROR_GETKEYS
,HBC_ERROR_LISTALL
,HBC_ERROR_REGION_STATS
+ ,HBC_ERROR_CREATE_SNAPSHOT_PARAM
+ ,HBC_ERROR_CREATE_SNAPSHOT_EXCEPTION
+ ,HBC_ERROR_DELETE_SNAPSHOT_PARAM
+ ,HBC_ERROR_DELETE_SNAPSHOT_EXCEPTION
+ ,HBC_ERROR_VERIFY_SNAPSHOT_PARAM
+ ,HBC_ERROR_VERIFY_SNAPSHOT_EXCEPTION
,HBC_LAST
} HBC_RetCode;
@@ -508,6 +514,9 @@ public:
HTableClient_JNI **outHtc);
NAArray<HbaseStr>* getStartKeys(NAHeap *heap, const char *tableName, bool useTRex);
NAArray<HbaseStr>* getEndKeys(NAHeap *heap, const char * tableName, bool useTRex);
+ HBC_RetCode createSnapshot( const NAString& tableName, const NAString& snapshotName);
+ HBC_RetCode deleteSnapshot( const NAString& snapshotName);
+ HBC_RetCode verifySnapshot( const NAString& tableName, const NAString& snapshotName, NABoolean & exist);
private:
// private default constructor
@@ -558,6 +567,9 @@ private:
,JM_HBC_CHECKANDDELETE_ROW
,JM_HBC_GETSTARTKEYS
,JM_HBC_GETENDKEYS
+ ,JM_HBC_CREATE_SNAPSHOT
+ ,JM_HBC_DELETE_SNAPSHOT
+ ,JM_HBC_VERIFY_SNAPSHOT
,JM_LAST
};
static jclass javaClass_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/SequenceFileReader.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/SequenceFileReader.cpp b/core/sql/executor/SequenceFileReader.cpp
index af31931..aff2945 100644
--- a/core/sql/executor/SequenceFileReader.cpp
+++ b/core/sql/executor/SequenceFileReader.cpp
@@ -455,13 +455,7 @@ static const char* const sfwErrorEnumStr[] =
,"JNI NewStringUTF() in hdfsExists()."
,"Java exception in hdfsExists()."
,"file already exists."
- ,"JNI NewStringUTF() in createSnapshot()."
- ,"Java exception in createSnapshot()."
- ,"JNI NewStringUTF() in deleteSnapshot()."
- ,"Java exception in deleteSnapshot()."
,"Java exception in release()."
- ,"JNI NewStringUTF() in verifySnapshot()."
- ,"Java exception in verifySnapshot()."
,"JNI NewStringUTF() in hdfsDeletePath()."
,"Java exception in hdfsDeletePath()."
};
@@ -526,12 +520,6 @@ SFW_RetCode SequenceFileWriter::init()
JavaMethods_[JM_HDFS_CLEAN_UNLOAD_PATH].jm_signature = "(Ljava/lang/String;)Z";
JavaMethods_[JM_HDFS_EXISTS].jm_name = "hdfsExists";
JavaMethods_[JM_HDFS_EXISTS].jm_signature = "(Ljava/lang/String;)Z";
- JavaMethods_[JM_CREATE_SNAPSHOT].jm_name = "createSnapshot";
- JavaMethods_[JM_CREATE_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
- JavaMethods_[JM_DELETE_SNAPSHOT].jm_name = "deleteSnapshot";
- JavaMethods_[JM_DELETE_SNAPSHOT].jm_signature = "(Ljava/lang/String;)Z";
- JavaMethods_[JM_VERIFY_SNAPSHOT].jm_name = "verifySnapshot";
- JavaMethods_[JM_VERIFY_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
JavaMethods_[JM_HDFS_DELETE_PATH].jm_name = "hdfsDeletePath";
JavaMethods_[JM_HDFS_DELETE_PATH].jm_signature = "(Ljava/lang/String;)Z";
@@ -808,121 +796,6 @@ SFW_RetCode SequenceFileWriter::hdfsCleanUnloadPath( const NAString& uldPath)
return SFW_OK;
}
-SFW_RetCode SequenceFileWriter::createSnapshot( const NAString& tableName, const NAString& snapshotName)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::createSnapshot(%s, %s) called.",
- tableName.data(), snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_tableName = jenv_->NewStringUTF(tableName.data());
- if (js_tableName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_CREATE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_PARAM;
- }
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_CREATE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_CREATE_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_CREATE_SNAPSHOT].methodID, js_tableName, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::createSnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION;
- }
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-SFW_RetCode SequenceFileWriter::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
- NABoolean & exist)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::verifySnapshot(%s, %s) called.",
- tableName.data(), snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_tableName = jenv_->NewStringUTF(tableName.data());
- if (js_tableName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_VERIFY_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_PARAM;
- }
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_VERIFY_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_VERIFY_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_VERIFY_SNAPSHOT].methodID, js_tableName, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::verifySnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
- }
-
- exist = jresult;
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-SFW_RetCode SequenceFileWriter::deleteSnapshot( const NAString& snapshotName)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::deleteSnapshot(%s) called.",
- snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_DELETE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_DELETE_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_DELETE_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_DELETE_SNAPSHOT].methodID, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::deleteSnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION;
- }
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-
SFW_RetCode SequenceFileWriter::hdfsMergeFiles( const NAString& srcPath,
const NAString& dstPath)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/SequenceFileReader.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/SequenceFileReader.h b/core/sql/executor/SequenceFileReader.h
index 9180de8..d950677 100644
--- a/core/sql/executor/SequenceFileReader.h
+++ b/core/sql/executor/SequenceFileReader.h
@@ -156,12 +156,6 @@ typedef enum {
,SFW_ERROR_HDFS_EXISTS_PARAM
,SFW_ERROR_HDFS_EXISTS_EXCEPTION
,SFW_ERROR_HDFS_EXISTS_FILE_EXISTS
- ,SFW_ERROR_CREATE_SNAPSHOT_PARAM
- ,SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION
- ,SFW_ERROR_DELETE_SNAPSHOT_PARAM
- ,SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION
- ,SFW_ERROR_VERIFY_SNAPSHOT_PARAM
- ,SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION
,SFW_ERROR_HDFS_DELETE_PATH_PARAM
,SFW_ERROR_HDFS_DELETE_PATH_EXCEPTION
,SFW_LAST
@@ -214,9 +208,6 @@ public:
SFW_RetCode hdfsCleanUnloadPath(const NAString& uldPath );
SFW_RetCode hdfsExists(const NAString& uldPath, NABoolean & exists );
SFW_RetCode hdfsClose();
- SFW_RetCode createSnapshot( const NAString& tableName, const NAString& snapshotName);
- SFW_RetCode deleteSnapshot( const NAString& snapshotName);
- SFW_RetCode verifySnapshot( const NAString& tableName, const NAString& snapshotName, NABoolean & exist);
SFW_RetCode release();
virtual char* getErrorText(SFW_RetCode errEnum);
@@ -237,9 +228,6 @@ private:
JM_HDFS_MERGE_FILES,
JM_HDFS_CLEAN_UNLOAD_PATH,
JM_HDFS_EXISTS,
- JM_CREATE_SNAPSHOT,
- JM_DELETE_SNAPSHOT,
- JM_VERIFY_SNAPSHOT,
JM_HDFS_DELETE_PATH,
JM_LAST
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseDefs.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseDefs.h b/core/sql/exp/ExpHbaseDefs.h
index 896735c..16690bf 100644
--- a/core/sql/exp/ExpHbaseDefs.h
+++ b/core/sql/exp/ExpHbaseDefs.h
@@ -130,6 +130,9 @@ typedef enum
HBASE_CLEANUP_HFILE_ERROR,
HBASE_INIT_HBLC_ERROR,
HBASE_RETRY_AGAIN,
+ HBASE_CREATE_SNAPSHOT_ERROR,
+ HBASE_DELETE_SNAPSHOT_ERROR,
+ HBASE_VERIFY_SNAPSHOT_ERROR,
HBASE_MAX_ERROR_NUM // keep this as the last element in enum list.
} HbaseError;
@@ -154,6 +157,9 @@ static const char * const hbaseErrorEnumStr[] =
"HBASE_DOBULK_LOAD_ERROR",
"HBASE_CLEANUP_HFILE_ERROR",
"HBASE_INIT_HBLC_ERROR",
+ "HBASE_CREATE_SNAPSHOT_ERROR",
+ "HBASE_DELETE_SNAPSHOT_ERROR",
+ "HBASE_VERIFY_SNAPSHOT_ERROR",
"HBASE_MAX_ERROR_NUM" // keep this as the last element in enum list.
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.cpp b/core/sql/exp/ExpHbaseInterface.cpp
index 0b5a77f..994d8fa 100644
--- a/core/sql/exp/ExpHbaseInterface.cpp
+++ b/core/sql/exp/ExpHbaseInterface.cpp
@@ -1588,3 +1588,48 @@ NAArray<HbaseStr> * ExpHbaseInterface_JNI::getClusterStats(Int32 &numEntries)
return regionStats;
}
+Lng32 ExpHbaseInterface_JNI::createSnapshot( const NAString& tableName, const NAString& snapshotName)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->createSnapshot(tableName, snapshotName);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_CREATE_SNAPSHOT_ERROR;
+}
+
+Lng32 ExpHbaseInterface_JNI::deleteSnapshot( const NAString& snapshotName)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->deleteSnapshot(snapshotName);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_DELETE_SNAPSHOT_ERROR;
+}
+
+Lng32 ExpHbaseInterface_JNI::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->verifySnapshot(tableName, snapshotName, exist);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_VERIFY_SNAPSHOT_ERROR;
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.h b/core/sql/exp/ExpHbaseInterface.h
index bf4a355..1d7728c 100644
--- a/core/sql/exp/ExpHbaseInterface.h
+++ b/core/sql/exp/ExpHbaseInterface.h
@@ -367,12 +367,17 @@ class ExpHbaseInterface : public NABasicObject
virtual Lng32 getRegionsNodeName(const HbaseStr& tblName,
Int32 partns,
ARRAY(const char *)& nodeNames) = 0;
-
// get regions and size
virtual NAArray<HbaseStr> *getRegionStats(const HbaseStr& tblName) = 0;
virtual NAArray<HbaseStr> *getClusterStats(Int32 &numEntries) = 0;
+ // Snapshots
+ virtual Lng32 createSnapshot( const NAString& tableName, const NAString& snapshotName) = 0;
+ virtual Lng32 deleteSnapshot( const NAString& snapshotName) = 0;
+ virtual Lng32 verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist) = 0;
+
protected:
enum
{
@@ -678,6 +683,11 @@ virtual Lng32 initHFileParams(HbaseStr &tblName,
virtual NAArray<HbaseStr>* getRegionStats(const HbaseStr& tblName);
virtual NAArray<HbaseStr>* getClusterStats(Int32 &numEntries);
+ virtual Lng32 createSnapshot( const NAString& tableName, const NAString& snapshotName);
+ virtual Lng32 deleteSnapshot( const NAString& snapshotName);
+ virtual Lng32 verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist);
+
private:
bool useTRex_;
HBaseClient_JNI* client_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
index c007f74..7bf02b2 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
@@ -1757,6 +1757,47 @@ public class HBaseClient {
return endKeys;
}
+ public boolean createSnapshot( String tableName, String snapshotName)
+ throws IOException
+ {
+ Admin admin = connection.getAdmin();
+ admin.snapshot(snapshotName, TableName.valueOf(tableName));
+ admin.close();
+ if (logger.isDebugEnabled()) logger.debug("HBaseClient.createSnapshot() - Snapshot created: " + snapshotName);
+ return true;
+ }
+
+ public boolean verifySnapshot( String tableName, String snapshotName)
+ throws IOException
+ {
+ Admin admin = connection.getAdmin();
+ List<SnapshotDescription> lstSnaps = admin.listSnapshots();
+ try
+ {
+ for (SnapshotDescription snpd : lstSnaps) {
+ if (snpd.getName().compareTo(snapshotName) == 0 &&
+ snpd.getTable().compareTo(tableName) == 0) {
+ if (logger.isDebugEnabled())
+ logger.debug("HBaseClient.verifySnapshot() - Snapshot verified: " + snapshotName);
+ return true;
+ }
+ }
+ } finally {
+ admin.close();
+ }
+ return false;
+ }
+
+ public boolean deleteSnapshot( String snapshotName)
+ throws MasterNotRunningException, IOException, SnapshotCreationException,
+ InterruptedException, ZooKeeperConnectionException, ServiceException
+ {
+ Admin admin = connection.getAdmin();
+ admin.deleteSnapshot(snapshotName);
+ admin.close();
+ if (logger.isDebugEnabled()) logger.debug("HBaseClient.deleteSnapshot() - Snapshot deleted: " + snapshotName);
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java b/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
index 4bd104f..9d51d37 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
@@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.trafodion.sql.HTableClient;
-//import org.trafodion.sql.HBaseClient;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.io.compress.CodecPool;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java b/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
index 28869fe..e1ee33c 100644
--- a/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
+++ b/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
@@ -34,13 +34,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.TableSnapshotScanner;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
@@ -76,8 +70,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
public class SequenceFileWriter {
static Logger logger = Logger.getLogger(SequenceFileWriter.class.getName());
- Configuration conf = null; // File system configuration
- private Connection connection;
+ static Configuration conf = null; // File system configuration
SequenceFile.Writer writer = null;
@@ -89,13 +82,12 @@ public class SequenceFileWriter {
/**
* Class Constructor
*/
- SequenceFileWriter() throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException
+ static {
+ conf = new Configuration(true);
+ }
+ SequenceFileWriter() throws IOException
{
- init("", "");
- conf = connection.getConfiguration();
- conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
}
-
public String open(String path) throws IOException {
Path filename = new Path(path);
@@ -304,56 +296,4 @@ public class SequenceFileWriter {
fs.delete(delPath, true);
return true;
}
-
- private boolean init(String zkServers, String zkPort) throws IOException
- , ServiceException
- {
- logger.debug("SequenceFileWriter.init(" + zkServers + ", " + zkPort + ") called.");
- connection = HBaseClient.getConnection();
- return true;
- }
-
- public boolean createSnapshot( String tableName, String snapshotName)
- throws IOException
- {
- Admin admin = connection.getAdmin();
- admin.snapshot(snapshotName, TableName.valueOf(tableName));
- admin.close();
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.createSnapshot() - Snapshot created: " + snapshotName);
- return true;
- }
-
- public boolean verifySnapshot( String tableName, String snapshotName)
- throws IOException
- {
- Admin admin = connection.getAdmin();
- List<SnapshotDescription> lstSnaps = admin.listSnapshots();
- try
- {
- for (SnapshotDescription snpd : lstSnaps)
- {
- if (snpd.getName().compareTo(snapshotName) == 0 &&
- snpd.getTable().compareTo(tableName) == 0)
- {
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.verifySnapshot() - Snapshot verified: " + snapshotName);
- return true;
- }
- }
- } finally {
- admin.close();
- }
- return false;
- }
-
- public boolean deleteSnapshot( String snapshotName)
- throws MasterNotRunningException, IOException, SnapshotCreationException,
- InterruptedException, ZooKeeperConnectionException, ServiceException
- {
- Admin admin = connection.getAdmin();
- admin.deleteSnapshot(snapshotName);
- admin.close();
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.deleteSnapshot() - Snapshot deleted: " + snapshotName);
- return true;
- }
-
}
[6/8] incubator-trafodion git commit: [TRAFODION-2351] Bulk load with
log error rows enhancements
Posted by se...@apache.org.
[TRAFODION-2351] Bulk load with log error rows enhancements
The LOAD command now outputs as follows:
Load with log error rows into selva.customer select * from hive.hive.customer ;
Task: LOAD Status: Started Object: TRAFODION.SELVA.CUSTOMER
Task: CLEANUP Status: Started Time: 2016-12-11 00:54:37.642
Task: CLEANUP Status: Ended Time: 2016-12-11 00:54:37.672
Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.030
Logging Location: /bulkload/logs/ERR_TRAFODION.SELVA.CUSTOMER_20161211_005437
Task: LOADING DATA Status: Started Time: 2016-12-11 00:54:37.672
Rows Processed: 99997
Task: LOADING DATA Status: Ended Time: 2016-12-11 00:54:58.296
Task: LOADING DATA Status: Ended Elapsed Time: 00:00:20.624
Task: COMPLETION Status: Started Time: 2016-12-11 00:54:58.296
Task: COMPLETION Status: Ended Time: 2016-12-11 00:54:59.521
Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.756
In addition, currently the "LOADING DATA" task status is shown as it happens. In future,
the task status will be shown as it happens for all tasks.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e0244ddc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e0244ddc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e0244ddc
Branch: refs/heads/master
Commit: e0244ddcafd6c7ce795cb6dd8f86df43695e7a2d
Parents: 76cb8b4
Author: selvaganesang <se...@esgyn.com>
Authored: Sun Dec 11 22:03:55 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Sun Dec 11 22:03:55 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 10 +-
core/sql/executor/ExExeUtilCommon.cpp | 24 +-
core/sql/executor/ExExeUtilLoad.cpp | 137 ++++++--
core/sql/executor/ExHbaseAccess.cpp | 23 +-
core/sql/executor/ExHbaseAccess.h | 3 +
core/sql/executor/ExHbaseIUD.cpp | 4 +-
core/sql/executor/ExHdfsScan.cpp | 4 +-
core/sql/regress/executor/EXPECTED140 | 61 ++--
core/sql/regress/executor/FILTER140 | 1 +
core/sql/regress/hive/EXPECTED015 | 230 +++++++------
core/sql/regress/hive/EXPECTED018 | 512 +++++++++++++++++------------
core/sql/regress/hive/FILTER015 | 3 +-
core/sql/regress/hive/FILTER018 | 3 +-
core/sql/sqlcomp/nadefaults.cpp | 2 +-
14 files changed, 630 insertions(+), 387 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index 9a8915f..7cfc232 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -252,6 +252,7 @@ class ExExeUtilTcb : public ex_tcb
const char * status,
const char * object,
char * outBuf,
+ NABoolean isET = FALSE,
char * timeBuf = NULL,
char * queryBuf = NULL,
char * sqlcodeBuf = NULL);
@@ -3450,7 +3451,7 @@ class ExExeUtilHBaseBulkLoadTcb : public ExExeUtilTcb
// Constructor
ExExeUtilHBaseBulkLoadTcb(const ComTdbExeUtil & exe_util_tdb,
ex_globals * glob = 0);
-
+ ~ExExeUtilHBaseBulkLoadTcb();
virtual short work();
ExExeUtilHBaseBulkLoadTdb & hblTdb() const
@@ -3461,6 +3462,8 @@ class ExExeUtilHBaseBulkLoadTcb : public ExExeUtilTcb
virtual short moveRowToUpQueue(const char * row, Lng32 len = -1,
short * rc = NULL, NABoolean isVarchar = TRUE);
+ short printLoggingLocation(int bufPos);
+
void setEndStatusMsg(const char * operation,
int bufPos = 0,
NABoolean withtime= FALSE);
@@ -3472,7 +3475,7 @@ class ExExeUtilHBaseBulkLoadTcb : public ExExeUtilTcb
NA_EIDPROC virtual ex_tcb_private_state * allocatePstates(
Lng32 &numElems, // inout, desired/actual elements
Lng32 &pstateLength); // out, length of one element
-
+ void setLoggingLocation();
private:
enum Step
{
@@ -3484,6 +3487,7 @@ class ExExeUtilHBaseBulkLoadTcb : public ExExeUtilTcb
LOAD_END_,
LOAD_END_ERROR_,
PREPARATION_,
+ LOADING_DATA_,
COMPLETE_BULK_LOAD_, //load incremental
POST_LOAD_CLEANUP_,
TRUNCATE_TABLE_,
@@ -3505,7 +3509,7 @@ class ExExeUtilHBaseBulkLoadTcb : public ExExeUtilTcb
Int64 rowsAffected_;
char statusMsgBuf_[BUFFER_SIZE];
ExpHbaseInterface * ehi_;
-
+ char *loggingLocation_;
short setCQDs();
short restoreCQDs();
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExExeUtilCommon.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilCommon.cpp b/core/sql/executor/ExExeUtilCommon.cpp
index ed0d302..a92e786 100644
--- a/core/sql/executor/ExExeUtilCommon.cpp
+++ b/core/sql/executor/ExExeUtilCommon.cpp
@@ -494,6 +494,7 @@ char * ExExeUtilTcb::getStatusString(const char * operation,
const char * status,
const char * object,
char * outBuf,
+ NABoolean isET,
char * timeBuf,
char * queryBuf,
char * errorBuf)
@@ -512,9 +513,9 @@ char * ExExeUtilTcb::getStatusString(const char * operation,
str_sprintf(outBuf, "Task: %s Status: %s Command: %s",
o, s, queryBuf);
}
- else if (timeBuf)
+ else if (timeBuf && isET)
{
- str_sprintf(outBuf, "Task: %s Status: %s ET: %s",
+ str_sprintf(outBuf, "Task: %s Status: %s Elapsed Time: %s",
o, s, timeBuf);
}
else if (errorBuf)
@@ -522,13 +523,18 @@ char * ExExeUtilTcb::getStatusString(const char * operation,
str_sprintf(outBuf, "Task: %s Status: %s Details: %s",
o, s, errorBuf);
}
+ else if (timeBuf)
+ {
+ str_sprintf(outBuf, "Task: %s Status: %s Time: %s",
+ o, s, timeBuf);
+ }
else
{
if (object)
- str_sprintf(outBuf, "Task: %s Status: %s Object: %s",
+ str_sprintf(outBuf, "Task: %s Status: %s Object: %s",
o, s, object);
else
- str_sprintf(outBuf, "Task: %s Status: %s",
+ str_sprintf(outBuf, "Task: %s Status: %s",
o, s);
}
@@ -549,6 +555,7 @@ short ExExeUtilTcb::executeQuery(char * task,
{
short retcode = 0;
char buf[BUFFER_SIZE];
+ char timeBuf[200];
while (1)
{
@@ -561,7 +568,8 @@ short ExExeUtilTcb::executeQuery(char * task,
elapsedTime_ = 0;
if (displayStartTime)
{
- getStatusString(task, "Started", object, buf);
+ getTimestampAsString(startTime_, timeBuf);
+ getStatusString(task, "Started", object, buf, FALSE, timeBuf);
if (moveRowToUpQueue(buf, 0, &rc))
return 1;
}
@@ -641,8 +649,12 @@ short ExExeUtilTcb::executeQuery(char * task,
if (displayEndTime)
{
char timeBuf[200];
+ getTimestampAsString(endTime_, timeBuf);
+ getStatusString(task, "Ended", object, buf, FALSE, timeBuf);
+ if (moveRowToUpQueue(buf, 0, &rc))
+ return 1;
getTimeAsString(elapsedTime_, timeBuf);
- getStatusString(task, "Ended", object, buf, timeBuf);
+ getStatusString(task, "Ended", object, buf, TRUE, timeBuf);
if (moveRowToUpQueue(buf, 0, &rc))
return 1;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index aea946b..a3a4423 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -1087,12 +1087,21 @@ ExExeUtilHBaseBulkLoadTcb::ExExeUtilHBaseBulkLoadTcb(
: ExExeUtilTcb( exe_util_tdb, NULL, glob),
step_(INITIAL_),
nextStep_(INITIAL_),
- rowsAffected_(0)
+ rowsAffected_(0),
+ loggingLocation_(NULL)
{
ehi_ = NULL;
qparent_.down->allocatePstate(this);
}
+ExExeUtilHBaseBulkLoadTcb::~ExExeUtilHBaseBulkLoadTcb()
+{
+ if (loggingLocation_ != NULL) {
+ NADELETEBASIC(loggingLocation_, getHeap());
+ loggingLocation_ = NULL;
+ }
+}
+
//////////////////////////////////////////////////////
// work() for ExExeUtilHbaseLoadTcb
//////////////////////////////////////////////////////
@@ -1141,7 +1150,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
break;
}
- if (setStartStatusMsgAndMoveToUpQueue("LOAD", &rc))
+ if (setStartStatusMsgAndMoveToUpQueue(" LOAD", &rc))
return rc;
if (hblTdb().getUpsertUsingLoad())
@@ -1177,7 +1186,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
case TRUNCATE_TABLE_:
{
- if (setStartStatusMsgAndMoveToUpQueue(" PURGE DATA",&rc))
+ if (setStartStatusMsgAndMoveToUpQueue(" PURGE DATA",&rc, 0, TRUE))
return rc;
// Set the parserflag to prevent privilege checks in purgedata
@@ -1216,7 +1225,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
}
step_ = LOAD_START_;
- setEndStatusMsg(" PURGE DATA");
+ setEndStatusMsg(" PURGE DATA", 0, TRUE);
}
break;
@@ -1267,8 +1276,8 @@ short ExExeUtilHBaseBulkLoadTcb::work()
case PRE_LOAD_CLEANUP_:
{
- if (setStartStatusMsgAndMoveToUpQueue(" CLEANUP", &rc))
- return rc;
+ if (setStartStatusMsgAndMoveToUpQueue(" CLEANUP", &rc, 0, TRUE))
+ return rc;
//Cleanup files
char * clnpQuery =
@@ -1300,12 +1309,12 @@ short ExExeUtilHBaseBulkLoadTcb::work()
if (hblTdb().getRebuildIndexes() || hblTdb().getHasUniqueIndexes())
step_ = DISABLE_INDEXES_;
- setEndStatusMsg(" CLEANUP");
+ setEndStatusMsg(" CLEANUP", 0, TRUE);
}
break;
case DISABLE_INDEXES_:
{
- if (setStartStatusMsgAndMoveToUpQueue(" DISABLE INDEXES", &rc))
+ if (setStartStatusMsgAndMoveToUpQueue(" DISABLE INDEXES", &rc, 0, TRUE))
return rc;
// disable indexes before starting the load preparation. load preparation phase will
@@ -1335,17 +1344,33 @@ short ExExeUtilHBaseBulkLoadTcb::work()
}
step_ = PREPARATION_;
- setEndStatusMsg(" DISABLE INDEXES");
+ setEndStatusMsg(" DISABLE INDEXES", 0, TRUE);
}
break;
case PREPARATION_:
{
+ short bufPos = 0;
if (!hblTdb().getUpsertUsingLoad())
{
- if (setStartStatusMsgAndMoveToUpQueue(" PREPARATION", &rc, 0, TRUE))
- return rc;
+ setLoggingLocation();
+ bufPos = printLoggingLocation(0);
+ if (setStartStatusMsgAndMoveToUpQueue(" LOADING DATA", &rc, bufPos, TRUE))
+ return rc;
+ else {
+ step_ = LOADING_DATA_;
+ return WORK_CALL_AGAIN;
+ }
+ }
+ else
+ step_ = LOADING_DATA_;
+ }
+ break;
+ case LOADING_DATA_:
+ {
+ if (!hblTdb().getUpsertUsingLoad())
+ {
if (hblTdb().getNoDuplicates())
cliRC = holdAndSetCQD("TRAF_LOAD_PREP_SKIP_DUPLICATES", "OFF");
else
@@ -1356,8 +1381,16 @@ short ExExeUtilHBaseBulkLoadTcb::work()
break;
}
+ if (loggingLocation_ != NULL)
+ cliRC = holdAndSetCQD("TRAF_LOAD_ERROR_LOGGING_LOCATION", loggingLocation_);
+ if (cliRC < 0)
+ {
+ step_ = LOAD_END_ERROR_;
+ break;
+ }
+
rowsAffected_ = 0;
- char * transQuery =hblTdb().ldQuery_;
+ char *loadQuery = hblTdb().ldQuery_;
if (ustatNonEmptyTable)
{
// If the ustat option was specified, but the table to be loaded
@@ -1365,11 +1398,11 @@ short ExExeUtilHBaseBulkLoadTcb::work()
// was added to the LOAD TRANSFORM statement when the original
// bulk load statement was parsed.
const char* sampleOpt = " WITH SAMPLE ";
- char* sampleOptPtr = strstr(transQuery, sampleOpt);
+ char* sampleOptPtr = strstr(loadQuery, sampleOpt);
if (sampleOptPtr)
memset(sampleOptPtr, ' ', strlen(sampleOpt));
}
- //printf("*** Load stmt is %s\n", transQuery);
+ //printf("*** Load stmt is %s\n",loadQuery);
// If the WITH SAMPLE clause is included, set the internal exe util
// parser flag to allow it.
@@ -1385,7 +1418,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
}
}
- cliRC = cliInterface()->executeImmediate(transQuery,
+ cliRC = cliInterface()->executeImmediate(loadQuery,
NULL,
NULL,
TRUE,
@@ -1393,7 +1426,6 @@ short ExExeUtilHBaseBulkLoadTcb::work()
if (parserFlagSet)
masterGlob->getStatement()->getContext()->resetSqlParserFlags(0x20000);
- transQuery = NULL;
if (cliRC < 0)
{
rowsAffected_ = 0;
@@ -1408,7 +1440,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
sprintf(statusMsgBuf_," Rows Processed: %ld %c",rowsAffected_, '\n' );
int len = strlen(statusMsgBuf_);
- setEndStatusMsg(" PREPARATION", len, TRUE);
+ setEndStatusMsg(" LOADING DATA", len, TRUE);
}
else
{
@@ -1513,7 +1545,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
case POPULATE_INDEXES_:
{
- if (setStartStatusMsgAndMoveToUpQueue(" POPULATE INDEXES", &rc))
+ if (setStartStatusMsgAndMoveToUpQueue(" POPULATE INDEXES", &rc, 0, TRUE))
return rc;
char * piQuery =
@@ -1550,7 +1582,7 @@ short ExExeUtilHBaseBulkLoadTcb::work()
case UPDATE_STATS_:
{
- if (setStartStatusMsgAndMoveToUpQueue(" UPDATE STATISTICS", &rc))
+ if (setStartStatusMsgAndMoveToUpQueue(" UPDATE STATISTICS", &rc, 0, TRUE))
return rc;
if (ustatNonEmptyTable)
@@ -1764,6 +1796,7 @@ short ExExeUtilHBaseBulkLoadTcb::restoreCQDs()
if (restoreCQD("TRAF_LOAD_MAX_ERROR_ROWS") < 0) { return -1;}
if (restoreCQD("TRAF_LOAD_ERROR_COUNT_TABLE") < 0) { return -1;}
if (restoreCQD("TRAF_LOAD_ERROR_COUNT_ID") < 0) { return -1; }
+ if (restoreCQD("TRAF_LOAD_ERROR_LOGGING_LOCATION") < 0) { return -1; }
return 0;
}
@@ -1778,6 +1811,19 @@ short ExExeUtilHBaseBulkLoadTcb::moveRowToUpQueue(const char * row, Lng32 len,
}
+short ExExeUtilHBaseBulkLoadTcb::printLoggingLocation(int bufPos)
+{
+ short retBufPos = bufPos;
+ if (hblTdb().getNoOutput())
+ return 0;
+ if (loggingLocation_ != NULL) {
+ str_sprintf(&statusMsgBuf_[bufPos], " Logging Location: %s", loggingLocation_);
+ retBufPos = strlen(statusMsgBuf_);
+ statusMsgBuf_[retBufPos] = '\n';
+ retBufPos++;
+ }
+ return retBufPos;
+}
short ExExeUtilHBaseBulkLoadTcb::setStartStatusMsgAndMoveToUpQueue(const char * operation,
@@ -1789,10 +1835,14 @@ short ExExeUtilHBaseBulkLoadTcb::setStartStatusMsgAndMoveToUpQueue(const char *
if (hblTdb().getNoOutput())
return 0;
+ char timeBuf[200];
+
if (withtime)
+ {
startTime_ = NA_JulianTimestamp();
-
- getStatusString(operation, "Started",hblTdb().getTableName(), &statusMsgBuf_[bufPos]);
+ getTimestampAsString(startTime_, timeBuf);
+ }
+ getStatusString(operation, "Started",hblTdb().getTableName(), &statusMsgBuf_[bufPos], FALSE, (withtime ? timeBuf : NULL));
return moveRowToUpQueue(statusMsgBuf_,0,rc);
}
@@ -1810,16 +1860,35 @@ void ExExeUtilHBaseBulkLoadTcb::setEndStatusMsg(const char * operation,
nextStep_ = step_;
step_ = RETURN_STATUS_MSG_;
+ Int64 elapsedTime;
if (withtime)
{
endTime_ = NA_JulianTimestamp();
- Int64 elapsedTime = endTime_ - startTime_;
-
+ elapsedTime = endTime_ - startTime_;
+ getTimestampAsString(endTime_, timeBuf);
+ getStatusString(operation, "Ended", hblTdb().getTableName(),&statusMsgBuf_[bufPos], FALSE, withtime ? timeBuf : NULL);
+ bufPos = strlen(statusMsgBuf_);
+ statusMsgBuf_[bufPos] = '\n';
+ bufPos++;
getTimeAsString(elapsedTime, timeBuf);
}
+ getStatusString(operation, "Ended", hblTdb().getTableName(),&statusMsgBuf_[bufPos], TRUE, withtime ? timeBuf : NULL);
+}
- getStatusString(operation, "Ended", hblTdb().getTableName(),&statusMsgBuf_[bufPos], withtime ? timeBuf : NULL);
-
+void ExExeUtilHBaseBulkLoadTcb::setLoggingLocation()
+{
+ char * loggingLocation = hblTdb().getLoggingLocation();
+ if (loggingLocation_ != NULL) {
+ NADELETEBASIC(loggingLocation_, getHeap());
+ loggingLocation_ = NULL;
+ }
+ if (loggingLocation != NULL) {
+ short logLen = strlen(loggingLocation);
+ char *tableName = hblTdb().getTableName();
+ short tableNameLen = strlen(tableName);
+ loggingLocation_ = new (getHeap()) char[logLen+tableNameLen+100];
+ ExHbaseAccessTcb::buildLoggingPath(loggingLocation, NULL, tableName, loggingLocation_);
+ }
}
////////////////////////////////////////////////////////////////////////
@@ -2525,11 +2594,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::setStartStatusMsgAndMoveToUpQueue(const char
if (hblTdb().getNoOutput())
return 0;
-
- if (withtime)
+ char timeBuf[200];
+ if (withtime) {
startTime_ = NA_JulianTimestamp();
-
- getStatusString(operation, "Started",NULL, &statusMsgBuf_[bufPos]);
+ getTimestampAsString(endTime_, timeBuf);
+ }
+ getStatusString(operation, "Started",NULL, &statusMsgBuf_[bufPos], FALSE, (withtime ? timeBuf : NULL));
return moveRowToUpQueue(statusMsgBuf_,0,rc);
}
@@ -2551,12 +2621,15 @@ void ExExeUtilHBaseBulkUnLoadTcb::setEndStatusMsg(const char * operation,
{
endTime_ = NA_JulianTimestamp();
Int64 elapsedTime = endTime_ - startTime_;
-
+ getTimestampAsString(endTime_, timeBuf);
+ getStatusString(operation, "Ended", hblTdb().getTableName(),&statusMsgBuf_[bufPos], FALSE, withtime ? timeBuf : NULL);
+ bufPos = strlen(statusMsgBuf_);
+ statusMsgBuf_[bufPos] = '\n';
+ bufPos++;
getTimeAsString(elapsedTime, timeBuf);
}
- getStatusString(operation, "Ended", NULL,&statusMsgBuf_[bufPos], withtime ? timeBuf : NULL);
-
+ getStatusString(operation, "Ended", NULL,&statusMsgBuf_[bufPos], TRUE, withtime ? timeBuf : NULL);
}
////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp
index a7b7f4e..d56d9df 100644
--- a/core/sql/executor/ExHbaseAccess.cpp
+++ b/core/sql/executor/ExHbaseAccess.cpp
@@ -3033,14 +3033,23 @@ void ExHbaseAccessTcb::setRowID(char *rowId, Lng32 rowIdLen)
memcpy(rowID_.val, rowId, rowIdLen);
}
-void ExHbaseAccessTcb::buildLoggingPath(
- const char * loggingLocation,
- char * logId,
- const char * tableName,
+void ExHbaseAccessTcb::buildLoggingFileName(
+ const char * currCmdLoggingLocation,
+ const char *tableName,
const char * loggingFileNamePrefix,
Lng32 instId,
char * loggingFileName)
{
+ sprintf(loggingFileName, "%s/%s_%s_%d",
+ currCmdLoggingLocation, loggingFileNamePrefix, tableName, instId);
+}
+
+void ExHbaseAccessTcb::buildLoggingPath(
+ const char *loggingLocation,
+ char * logId,
+ const char * tableName,
+ char * currCmdLoggingLocation)
+{
time_t t;
char logId_tmp[30];
@@ -3050,9 +3059,9 @@ void ExHbaseAccessTcb::buildLoggingPath(
struct tm * curgmtime = gmtime(&t);
strftime(logId_tmp, sizeof(logId_tmp)-1, "%Y%m%d_%H%M%S", curgmtime);
logId = logId_tmp;
- }
- sprintf(loggingFileName, "%s/ERR_%s/%s_%s_%d",
- loggingLocation, logId, tableName, loggingFileNamePrefix, instId);
+ }
+ sprintf(currCmdLoggingLocation, "%s/ERR_%s_%s",
+ loggingLocation, tableName, logId);
}
void ExHbaseAccessTcb::handleException(NAHeap *heap,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index 03cb052..a1e25c2 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -184,6 +184,9 @@ public:
static void buildLoggingPath(const char * loggingLocation,
char *logId,
const char *tableName,
+ char * currCmdLoggingLocation);
+ static void buildLoggingFileName(const char * currCmdLoggingLocation,
+ const char *tableName,
const char * loggingFileNamePrefix,
Lng32 instId,
char * loggingFileName);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp
index d3106f3..6b80f3f 100644
--- a/core/sql/executor/ExHbaseIUD.cpp
+++ b/core/sql/executor/ExHbaseIUD.cpp
@@ -1076,8 +1076,8 @@ ExHbaseAccessBulkLoadPrepSQTcb::ExHbaseAccessBulkLoadPrepSQTcb(
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
- ExHbaseAccessTcb::buildLoggingPath(((ExHbaseAccessTdb &)hbaseAccessTdb).getLoggingLocation(),
- (char *)((ExHbaseAccessTdb &)hbaseAccessTdb).getErrCountRowId(),
+ ExHbaseAccessTcb::buildLoggingFileName(((ExHbaseAccessTdb &)hbaseAccessTdb).getLoggingLocation(),
+ // (char *)((ExHbaseAccessTdb &)hbaseAccessTdb).getErrCountRowId(),
((ExHbaseAccessTdb &)hbaseAccessTdb).getTableName(),
"traf_upsert_err",
fileNum,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHdfsScan.cpp b/core/sql/executor/ExHdfsScan.cpp
index 8e43950..9ce17d5 100644
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@ -187,8 +187,8 @@ ExHdfsScanTcb::ExHdfsScanTcb(
registerResizeSubtasks();
Lng32 fileNum = getGlobals()->castToExExeStmtGlobals()->getMyInstanceNumber();
- ExHbaseAccessTcb::buildLoggingPath(((ExHdfsScanTdb &)hdfsScanTdb).getLoggingLocation(),
- (char *)((ExHdfsScanTdb &)hdfsScanTdb).getErrCountRowId(),
+ ExHbaseAccessTcb::buildLoggingFileName(((ExHdfsScanTdb &)hdfsScanTdb).getLoggingLocation(),
+ // (char *)((ExHdfsScanTdb &)hdfsScanTdb).getErrCountRowId(),
((ExHdfsScanTdb &)hdfsScanTdb).tableName(),
"hive_scan_err",
fileNum,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/executor/EXPECTED140
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/EXPECTED140 b/core/sql/regress/executor/EXPECTED140
index 390c05d..e29725e 100644
--- a/core/sql/regress/executor/EXPECTED140
+++ b/core/sql/regress/executor/EXPECTED140
@@ -123,14 +123,17 @@
+> , SS_NET_PAID_INC_TAX
+> , SS_NET_PROFIT
+>from hive.hive.store_sales where ss_sold_date_sk is not null;
-Task: LOAD Status: Started Object: TRAFODION.SCH.T140C
-Task: CLEANUP Status: Started Object: TRAFODION.SCH.T140C
-Task: CLEANUP Status: Ended Object: TRAFODION.SCH.T140C
-Task: PREPARATION Status: Started Object: TRAFODION.SCH.T140C
+Task: LOAD Status: Started Object: TRAFODION.SCH.T140C
+Task: CLEANUP Status: Started Time: 2016-12-11 21:58:01.242
+Task: CLEANUP Status: Ended Time: 2016-12-11 21:58:01.272
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.030
+Task: LOADING DATA Status: Started Time: 2016-12-11 21:58:01.272
Rows Processed: 2750311
-Task: PREPARATION Status: Ended ET: 00:01:24.119
-Task: COMPLETION Status: Started Object: TRAFODION.SCH.T140C
-Task: COMPLETION Status: Ended ET: 00:00:02.219
+Task: LOADING DATA Status: Ended Time: 2016-12-11 21:58:29.607
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:28.335
+Task: COMPLETION Status: Started Time: 2016-12-11 21:58:29.608
+Task: COMPLETION Status: Ended Time: 2016-12-11 21:58:30.463
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.855
--- 2750311 row(s) loaded.
>>update statistics for table t140c on every column sample;
@@ -151,7 +154,7 @@ Task: COMPLETION Status: Ended ET: 00:00:02.219
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158264313307
+PLAN_ID .................. 212348253556102007
ROWS_OUT ................ 33
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select a from t140 where b>500;
@@ -189,7 +192,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.A
@@ -237,7 +240,7 @@ A
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158264999300
+PLAN_ID .................. 212348253556356455
ROWS_OUT ................ 33
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an from t140b where b<=200;
@@ -275,7 +278,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398174249
+ ObjectUIDs ............. 3170650258679964269
select_list ............ TRAFODION.SCH.T140B.AN
@@ -310,7 +313,7 @@ DESCRIPTION
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158265214353
+PLAN_ID .................. 212348253556524433
ROWS_OUT ................ 33
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an from t140 where b<=200;
@@ -348,7 +351,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN
@@ -394,7 +397,7 @@ AN
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158265449379
+PLAN_ID .................. 212348253556705275
ROWS_OUT ................ 10
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an from t140 where b=200 and an is not null;
@@ -432,7 +435,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN
input_variables ........ %(200)
@@ -476,7 +479,7 @@ AN
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158265858457
+PLAN_ID .................. 212348253556934945
ROWS_OUT ................ 67
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an, a from t140 where b!=500;
@@ -514,7 +517,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN, TRAFODION.SCH.T140.A
@@ -594,7 +597,7 @@ AN
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158266176554
+PLAN_ID .................. 212348253557164907
ROWS_OUT ................ 10
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an from t140 where bn=201 and an is not null;
@@ -632,7 +635,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN
input_variables ........ %(201)
@@ -675,7 +678,7 @@ AN
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158266394629
+PLAN_ID .................. 212348253557343978
ROWS_OUT ................ 67
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an, a from t140 where bn!=501;
@@ -713,7 +716,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN, TRAFODION.SCH.T140.A
@@ -808,7 +811,7 @@ A
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158266709439
+PLAN_ID .................. 212348253557565585
ROWS_OUT ................ 11
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select an from t140 where an between 20 and 40;
@@ -846,7 +849,7 @@ DESCRIPTION
SCHEMA ................. TRAFODION.SCH
TRAF_ALIGNED_ROW_FORMAT OFF
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398173909
+ ObjectUIDs ............. 3170650258679963857
select_list ............ TRAFODION.SCH.T140.AN
@@ -913,7 +916,7 @@ AN
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158267230725
+PLAN_ID .................. 212348253557816401
ROWS_OUT ................. 1
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select avg(a) from t140b;
@@ -953,7 +956,7 @@ DESCRIPTION
PARALLEL_NUM_ESPS ...... 1
HBASE_DOP_PARALLEL_SCAN 2
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398174249
+ ObjectUIDs ............. 3170650258679964269
select_list ............ cast(cast((cast((cast((cast(sum(TRAFODION.SCH.T140B.
A)) * cast(10000 ...0))) / cast(count(1 )))) /
cast(10000 ...0))))
@@ -1013,7 +1016,7 @@ DESCRIPTION
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158267953185
+PLAN_ID .................. 212348253558056527
ROWS_OUT ................. 1
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select avg(a) from t140b;
@@ -1053,7 +1056,7 @@ DESCRIPTION
PARALLEL_NUM_ESPS ...... 1
HBASE_DOP_PARALLEL_SCAN 1
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398174249
+ ObjectUIDs ............. 3170650258679964269
select_list ............ cast(cast((cast((cast((cast(sum(TRAFODION.SCH.T140B.
A)) * cast(10000 ...0))) / cast(count(1 )))) /
cast(10000 ...0))))
@@ -1109,7 +1112,7 @@ DESCRIPTION
------------------------------------------------------------------ PLAN SUMMARY
MODULE_NAME .............. DYNAMICALLY COMPILED
STATEMENT_NAME ........... NOT NAMED
-PLAN_ID .................. 212341158267953185
+PLAN_ID .................. 212348253558056527
ROWS_OUT ................. 1
EST_TOTAL_COST ........... 0.01
STATEMENT ................ select avg(a) from t140b;
@@ -1149,7 +1152,7 @@ DESCRIPTION
PARALLEL_NUM_ESPS ...... 1
HBASE_DOP_PARALLEL_SCAN 1
GENERATE_EXPLAIN ....... ON
- ObjectUIDs ............. 6419990294398174249
+ ObjectUIDs ............. 3170650258679964269
select_list ............ cast(cast((cast((cast((cast(sum(TRAFODION.SCH.T140B.
A)) * cast(10000 ...0))) / cast(count(1 )))) /
cast(10000 ...0))))
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/executor/FILTER140
----------------------------------------------------------------------
diff --git a/core/sql/regress/executor/FILTER140 b/core/sql/regress/executor/FILTER140
index 37f371f..9363956 100755
--- a/core/sql/regress/executor/FILTER140
+++ b/core/sql/regress/executor/FILTER140
@@ -42,5 +42,6 @@ s/^\(\.*PREPARATION[ ]Status:[ ]*\) .*/removed/g
s/^\([ ]*Task:[ ]COMPLETION[ ]Status:[ ]*\) .*/removed/g
s/.*PREPARATION.*/removed/
s/.*COMPLETION.*/removed/
+s/Time:.*/Time:@time@/
s/VOLATILE_SCHEMA_MXID[0-9]*/VOLATILE_SCHEMA_00/
" $fil
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/e0244ddc/core/sql/regress/hive/EXPECTED015
----------------------------------------------------------------------
diff --git a/core/sql/regress/hive/EXPECTED015 b/core/sql/regress/hive/EXPECTED015
index 1ede641..65c998b 100644
--- a/core/sql/regress/hive/EXPECTED015
+++ b/core/sql/regress/hive/EXPECTED015
@@ -205,14 +205,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
--- SQL operation complete.
>>execute s;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
+Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
+Task: CLEANUP Status: Started Time: 2016-12-11 01:53:25.703
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:53:25.734
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.032
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:53:25.734
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.083
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.344
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:53:25.800
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.065
+Task: COMPLETION Status: Started Time: 2016-12-11 01:53:25.800
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:53:26.692
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.892
--- 5 row(s) loaded.
>>
@@ -229,14 +232,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
--- SQL operation complete.
>>execute s;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
+Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
+Task: CLEANUP Status: Started Time: 2016-12-11 01:53:27.582
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:53:27.614
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.032
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:53:27.614
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.068
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.151
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:53:27.678
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.064
+Task: COMPLETION Status: Started Time: 2016-12-11 01:53:27.678
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:53:27.902
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.224
--- 5 row(s) loaded.
>>
@@ -254,14 +260,18 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
--- SQL operation complete.
>>execute s;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
+Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
+Task: CLEANUP Status: Started Time: 2016-12-11 01:53:28.873
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:53:28.894
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.021
+ Logging Location: /bulkload/logs/ERR_TRAFODION.HBASE.T015T2_20161211_015328
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:53:28.894
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.075
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.275
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:53:29.781
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.113
+Task: COMPLETION Status: Started Time: 2016-12-11 01:53:29.782
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:53:29.592
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.585
--- 5 row(s) loaded.
>>
@@ -282,16 +292,20 @@ A
--- 3 row(s) selected.
>>load with truncate table into t015t2 select * from t015t1;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
-Task: PURGE DATA Status: Started Object: TRAFODION.HBASE.T015T2
-Task: PURGE DATA Status: Ended Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.T015T2
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.T015T2
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.T015T2
+Task: LOAD Status: Started Object: TRAFODION.HBASE.T015T2
+Task: PURGE DATA Status: Started Time: 2016-12-11 01:53:30.251
+Task: PURGE DATA Status: Ended Time: 2016-12-11 01:53:35.963
+Task: PURGE DATA Status: Ended Elapsed Time: 00:00:05.711
+Task: CLEANUP Status: Started Time: 2016-12-11 01:53:36.273
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:53:36.873
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.600
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:53:36.873
Rows Processed: 5
-Task: PREPARATION Status: Ended ET: 00:00:00.068
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.T015T2
-Task: COMPLETION Status: Ended ET: 00:00:00.279
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:53:36.947
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.073
+Task: COMPLETION Status: Started Time: 2016-12-11 01:53:36.947
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:53:37.322
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.376
--- 5 row(s) loaded.
>>
@@ -457,14 +471,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>
>>load into customer_address
+>select * from hive.hive.customer_address where ca_address_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
+Task: CLEANUP Status: Started Time: 2016-12-11 01:54:22.412
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:54:22.434
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.022
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:54:22.434
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:07.833
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS
-Task: COMPLETION Status: Ended ET: 00:00:00.608
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:54:28.141
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:05.707
+Task: COMPLETION Status: Started Time: 2016-12-11 01:54:28.141
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:54:28.602
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.461
--- 5000 row(s) loaded.
>>
@@ -539,14 +556,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>
>>load into customer_address_NOPK
+>select * from hive.hive.customer_address where ca_address_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
+Task: CLEANUP Status: Started Time: 2016-12-11 01:54:30.612
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:54:30.634
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.022
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:54:30.634
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:00.953
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_ADDRESS_NOPK
-Task: COMPLETION Status: Ended ET: 00:00:00.399
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:54:31.365
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:00.731
+Task: COMPLETION Status: Started Time: 2016-12-11 01:54:31.365
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:54:31.702
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.337
--- 5000 row(s) loaded.
>>
@@ -640,14 +660,17 @@ ESP_EXCHANGE ============================== SEQ_NO 2 ONLY CHILD 1
>>
>>load into customer_demographics
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: CLEANUP Status: Started Time: 2016-12-11 01:54:35.222
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:54:35.245
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.023
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:54:35.245
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:03.825
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:00.594
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:54:36.722
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:01.477
+Task: COMPLETION Status: Started Time: 2016-12-11 01:54:36.722
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:54:37.692
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:00.970
--- 5000 row(s) loaded.
>>
@@ -744,14 +767,17 @@ ESP_EXCHANGE ============================== SEQ_NO 2 ONLY CHILD 1
>>
>>load into customer_demographics_salt
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: CLEANUP Status: Started Time: 2016-12-11 01:54:42.213
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:54:42.235
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.022
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:54:42.235
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:06.638
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:01.913
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:54:45.254
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:02.790
+Task: COMPLETION Status: Started Time: 2016-12-11 01:54:45.255
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:54:46.273
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.248
--- 5000 row(s) loaded.
>>
@@ -1256,14 +1282,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>
>>load into customer_demographics
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: CLEANUP Status: Started Time: 2016-12-11 01:56:43.352
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:56:43.375
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.023
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:56:43.375
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:06.553
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:01.562
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:56:48.504
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:04.675
+Task: COMPLETION Status: Started Time: 2016-12-11 01:56:48.504
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:56:49.912
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.862
--- 5000 row(s) loaded.
>>
@@ -1297,18 +1326,23 @@ Task: COMPLETION Status: Ended ET: 00:00:01.562
>>
>>load with rebuild indexes into customer_demographics
+>select * from hive.hive.customer_demographics where cd_demo_sk > 5000 and cd_demo_sk <= 6000 ;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
+Task: CLEANUP Status: Started Time: 2016-12-11 01:56:53.213
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:56:53.887
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.674
+Task: DISABLE INDEXE Status: Started Time: 2016-12-11 01:56:53.887
+Task: DISABLE INDEXE Status: Ended Time: 2016-12-11 01:57:07.143
+Task: DISABLE INDEXE Status: Ended Elapsed Time: 00:00:13.256
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:57:07.143
Rows Processed: 1000
-Task: PREPARATION Status: Ended ET: 00:00:03.781
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: COMPLETION Status: Ended ET: 00:00:01.479
-Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS
-Task: POPULATE INDEX Status: Ended ET: 00:00:13.703
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:57:09.535
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:02.391
+Task: COMPLETION Status: Started Time: 2016-12-11 01:57:09.535
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:57:11.123
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.588
+Task: POPULATE INDEX Status: Started Time: 2016-12-11 01:57:11.123
+Task: POPULATE INDEX Status: Ended Time: 2016-12-11 01:57:19.652
+Task: POPULATE INDEX Status: Ended Elapsed Time: 00:00:08.529
--- 1000 row(s) loaded.
>>
@@ -1346,18 +1380,23 @@ Task: POPULATE INDEX Status: Ended ET: 00:00:13.703
>>---------------------------
>>load into customer_demographics_salt
+>select * from hive.hive.customer_demographics where cd_demo_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: DISABLE INDEXE Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: DISABLE INDEXE Status: Ended Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: LOAD Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
+Task: CLEANUP Status: Started Time: 2016-12-11 01:57:41.352
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:57:42.135
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.784
+Task: DISABLE INDEXE Status: Started Time: 2016-12-11 01:57:42.135
+Task: DISABLE INDEXE Status: Ended Time: 2016-12-11 01:57:55.304
+Task: DISABLE INDEXE Status: Ended Elapsed Time: 00:00:13.168
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:57:55.304
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:05.478
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: COMPLETION Status: Ended ET: 00:00:02.063
-Task: POPULATE INDEX Status: Started Object: TRAFODION.HBASE.CUSTOMER_DEMOGRAPHICS_SALT
-Task: POPULATE INDEX Status: Ended ET: 00:00:15.035
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:57:59.470
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:04.166
+Task: COMPLETION Status: Started Time: 2016-12-11 01:57:59.470
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:58:00.723
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.253
+Task: POPULATE INDEX Status: Started Time: 2016-12-11 01:58:00.723
+Task: POPULATE INDEX Status: Ended Time: 2016-12-11 01:58:09.672
+Task: POPULATE INDEX Status: Ended Elapsed Time: 00:00:08.950
--- 5000 row(s) loaded.
>>
@@ -1526,14 +1565,17 @@ LC RC OP OPERATOR OPT DESCRIPTION CARD
>>
>>load into "customer_address_delim"
+>select * from hive.hive.customer_address where ca_address_sk <= 5000;
-Task: LOAD Status: Started Object: TRAFODION.HBASE."customer_address_delim"
-Task: CLEANUP Status: Started Object: TRAFODION.HBASE."customer_address_delim"
-Task: CLEANUP Status: Ended Object: TRAFODION.HBASE."customer_address_delim"
-Task: PREPARATION Status: Started Object: TRAFODION.HBASE."customer_address_delim"
+Task: LOAD Status: Started Object: TRAFODION.HBASE."customer_address_delim"
+Task: CLEANUP Status: Started Time: 2016-12-11 01:58:54.252
+Task: CLEANUP Status: Ended Time: 2016-12-11 01:58:54.934
+Task: CLEANUP Status: Ended Elapsed Time: 00:00:00.682
+Task: LOADING DATA Status: Started Time: 2016-12-11 01:58:54.934
Rows Processed: 5000
-Task: PREPARATION Status: Ended ET: 00:00:03.841
-Task: COMPLETION Status: Started Object: TRAFODION.HBASE."customer_address_delim"
-Task: COMPLETION Status: Ended ET: 00:00:01.934
+Task: LOADING DATA Status: Ended Time: 2016-12-11 01:58:57.258
+Task: LOADING DATA Status: Ended Elapsed Time: 00:00:02.324
+Task: COMPLETION Status: Started Time: 2016-12-11 01:58:57.258
+Task: COMPLETION Status: Ended Time: 2016-12-11 01:58:58.433
+Task: COMPLETION Status: Ended Elapsed Time: 00:00:01.175
--- 5000 row(s) loaded.
>>
[2/8] incubator-trafodion git commit: Merge branch 'master' of
git://git.apache.org/incubator-trafodion into trafodion-2356
Posted by se...@apache.org.
Merge branch 'master' of git://git.apache.org/incubator-trafodion into trafodion-2356
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/e0dab3ee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/e0dab3ee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/e0dab3ee
Branch: refs/heads/master
Commit: e0dab3ee05c39b441de9f04fe3e04654b03cf8d4
Parents: 055f7bb a596155
Author: selvaganesang <se...@esgyn.com>
Authored: Sat Nov 19 00:40:11 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Sat Nov 19 00:40:11 2016 +0000
----------------------------------------------------------------------
install/installer/traf_start | 13 ++++++-------
1 file changed, 6 insertions(+), 7 deletions(-)
----------------------------------------------------------------------