You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2018/02/16 20:00:42 UTC
[9/9] trafodion git commit: Merge PR 1417 [TRAFODION-2917] Refactor
Trafodion implementation of hdfs scan for text formatted hive tables
Merge PR 1417 [TRAFODION-2917] Refactor Trafodion implementation of hdfs scan for text formatted hive tables
Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/de835767
Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/de835767
Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/de835767
Branch: refs/heads/master
Commit: de8357677276e3987f24b550da5b7d0fc2b8200c
Parents: 087af70 cc43432
Author: selvaganesang <se...@apache.org>
Authored: Fri Feb 16 19:59:39 2018 +0000
Committer: selvaganesang <se...@apache.org>
Committed: Fri Feb 16 19:59:39 2018 +0000
----------------------------------------------------------------------
core/sql/comexe/ComTdbHdfsScan.h | 10 +-
core/sql/executor/ExExeUtil.h | 3 +-
core/sql/executor/ExExeUtilCli.cpp | 1 -
core/sql/executor/ExExeUtilGet.cpp | 16 +-
core/sql/executor/ExExeUtilLoad.cpp | 53 +-
core/sql/executor/ExExeUtilMisc.cpp | 1 -
core/sql/executor/ExFastTransport.cpp | 100 ++-
core/sql/executor/ExFastTransport.h | 5 +-
core/sql/executor/ExHbaseAccess.cpp | 53 +-
core/sql/executor/ExHbaseAccess.h | 32 +-
core/sql/executor/ExHbaseIUD.cpp | 58 +-
core/sql/executor/ExHdfsScan.cpp | 378 ++++++---
core/sql/executor/ExHdfsScan.h | 69 +-
core/sql/executor/HBaseClient_JNI.cpp | 147 +---
core/sql/executor/HBaseClient_JNI.h | 23 +-
core/sql/executor/HdfsClient_JNI.cpp | 763 +++++++++++++++++++
core/sql/executor/HdfsClient_JNI.h | 192 +++++
core/sql/executor/JavaObjectInterface.cpp | 24 +-
core/sql/executor/JavaObjectInterface.h | 24 +-
core/sql/executor/OrcFileReader.cpp | 3 +-
core/sql/executor/SequenceFileReader.cpp | 304 +-------
core/sql/executor/SequenceFileReader.h | 30 -
core/sql/exp/ExpErrorEnums.h | 1 +
core/sql/exp/ExpHbaseInterface.cpp | 93 +--
core/sql/exp/ExpHbaseInterface.h | 18 +-
core/sql/exp/ExpLOBinterface.h | 13 -
core/sql/generator/GenRelScan.cpp | 3 +
core/sql/nskgmake/executor/Makefile | 1 +
core/sql/qmscommon/QRLogger.cpp | 1 +
core/sql/qmscommon/QRLogger.h | 1 +
core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 1 -
core/sql/sqlcomp/DefaultConstants.h | 3 +
core/sql/sqlcomp/nadefaults.cpp | 2 +
.../main/java/org/trafodion/sql/HDFSClient.java | 394 ++++++++++
.../main/java/org/trafodion/sql/HdfsScan.java | 289 +++++++
.../org/trafodion/sql/SequenceFileWriter.java | 154 ----
.../org/trafodion/sql/TrafConfiguration.java | 1 +
37 files changed, 2244 insertions(+), 1020 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExExeUtilMisc.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExFastTransport.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/executor/ExHdfsScan.cpp
----------------------------------------------------------------------
diff --cc core/sql/executor/ExHdfsScan.cpp
index 7dd933b,c36270e..2b73feb
--- a/core/sql/executor/ExHdfsScan.cpp
+++ b/core/sql/executor/ExHdfsScan.cpp
@@@ -280,9 -307,15 +307,15 @@@ void ExHdfsScanTcb::freeResources(
}
deallocateRuntimeRanges();
if (lobGlob_) {
- ExpLOBinterfaceCleanup(lobGlob_, (NAHeap *)getGlobals()->getDefaultHeap());
+ ExpLOBinterfaceCleanup(lobGlob_);
lobGlob_ = NULL;
}
+ if (hdfsClient_ != NULL)
+ NADELETE(hdfsClient_, HdfsClient, getHeap());
+ if (logFileHdfsClient_ != NULL)
+ NADELETE(logFileHdfsClient_, HdfsClient, getHeap());
+ if (hdfsScan_ != NULL)
+ NADELETE(hdfsScan_, HdfsScan, getHeap());
}
NABoolean ExHdfsScanTcb::needStatsEntry()
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --cc core/sql/exp/ExpLOBinterface.h
index c8a8591,b98d2b4..8194859
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@@ -342,42 -342,8 +342,29 @@@ Lng32 ExpLOBInterfaceGetLobLength(ExLob
Int64 &outLobLen
);
+Lng32 ExpLOBInterfaceGetFileName(ExLobGlobals * exLobGlob,
+ char * lobName,
+ char * lobLoc,
+ Lng32 lobType,
+ char * lobHdfsServer,
+ Lng32 lobHdfsPort,
+ Int32 handleLen,
+ char * lobHandle,
+ char * outFileName,
+ Int32 &outFileLen);
+Lng32 ExpLOBInterfaceGetOffset(ExLobGlobals * exLobGlob,
+ char * lobName,
+ char * lobLoc,
+ Lng32 lobType,
+ char * lobHdfsServer,
+ Lng32 lobHdfsPort,
+ Int32 handleLen,
+ char * lobHandle,
+ Int64 &outLobOffset
+
+ );
- /*
- class HdfsFileInfo
- {
- public:
- char * fileName() { return fileName_; }
- Int64 getStartOffset() { return startOffset_; }
- Int64 getBytesToRead() { return bytesToRead_; }
- Lng32 entryNum_; // 0 based, first entry is entry num 0.
- NABasicPtr fileName_;
- Int64 startOffset_;
- Int64 bytesToRead_;
- };
- */
#endif
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/qmscommon/QRLogger.cpp
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/qmscommon/QRLogger.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/trafodion/blob/de835767/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------