You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by sa...@apache.org on 2016/03/09 19:44:01 UTC
[03/11] incubator-trafodion git commit: Changes for performing GC.
Changes to pass in hdfsServer and port via CQD and remove hardcoding. Changes
to pass in lobLocation via CQD and remove hardcoding. Changes to remove all
mxlobsrvr process functionality
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBaccess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBaccess.h b/core/sql/exp/ExpLOBaccess.h
index 6062775..bcdba44 100644
--- a/core/sql/exp/ExpLOBaccess.h
+++ b/core/sql/exp/ExpLOBaccess.h
@@ -46,7 +46,7 @@
#include "ExpLOBstats.h"
#include "ExpLOBinterface.h"
#include "ComSmallDefs.h"
-
+#include "Globals.h"
#include <seabed/ms.h>
#include <seabed/fs.h>
@@ -79,6 +79,79 @@ using namespace std;
#define LOB_CURSOR_PREFETCH_BYTES_MAX (1 << 27) // 128MB
class ExLobGlobals;
+// This class defines the request used to construct the message to send over
+// to the mxlobsrvr process. It's currently not used. All lob functionailty is
+// now in te master process. We are retaining it here for furture use.
+class ExLobRequest
+{
+ public:
+ ExLobRequest();
+ ~ExLobRequest();
+ Ex_Lob_Error send();
+
+ void setValues(char *descFileName, Int64 descNumIn, Int64 handleInLen,
+ char *handleIn, LobsStorage storage, Int64 transId,
+ SB_Transid_Type transIdBig,
+ SB_Transseq_Type transStartId,
+ char *blackBox, Int64 blackBoxLen);
+ void getValues(Int64 &descNumOut, Int64 &handleOutLen,
+ char *handleOut, Ex_Lob_Error &requestStatus,
+ Int64 &cliError,
+ char *blackBox, Int64 &blackBoxLen);
+
+ Int64 getDescNumIn() { return descNumIn_; }
+ void setDescNumIn(Int64 descNum) { descNumIn_ = descNum; }
+ Int64 getDescNumOut() { return descNumOut_; }
+ void setDescNumOut(Int64 descNum) { descNumOut_ = descNum; }
+ Int64 getDataOffset() { return dataOffset_; }
+ void setDataOffset(int dataOffset) { dataOffset_ = dataOffset; }
+ LobsRequest getType() { return type_; }
+ void setType(LobsRequest type) { type_ = type; }
+ char *getDescFileName() { return descFileName_; }
+ Int64 getOperLen() { return operLen_; }
+ void setOperLen(Int64 len) { operLen_ = len; }
+ void log();
+ void setError(Ex_Lob_Error err) { error_ = err; }
+ Ex_Lob_Error getError() { return error_; }
+ Ex_Lob_Error getStatus() { return status_; }
+ char *getHandleIn() { return handleIn_; }
+ Int64 getHandleInLen() { return handleInLen_; }
+ char *getHandleOut() { return handleOut_; }
+ void setHandleOutLen(Lng32 len) { handleOutLen_ = len; }
+ void setCliError(int cliErr) { cliError_ = cliErr; }
+ int getCliError() { return (int)cliError_; }
+ Int64 getTransId() { return transId_; }
+ SB_Transid_Type getTransIdBig() { return transIdBig_; }
+ SB_Transseq_Type getTransStartId() { return transStartId_; }
+ Int64 getBlackBoxLen() { return blackBoxLen_; };
+ void setBlackBoxLen(Int64 len) { blackBoxLen_ = len; }
+ char *getBlackBox() { return blackBox_; }
+
+ void incrReqNum() { reqNum_++; }
+ Int64 getReqNum() { return reqNum_; }
+
+ private:
+ Int64 reqNum_;
+ Int64 descNumIn_;
+ Int64 descNumOut_;
+ char handleIn_[MAX_HANDLE_IN_LEN];
+ Int64 handleInLen_;
+ char handleOut_[MAX_HANDLE_OUT_LEN];
+ Int64 handleOutLen_;
+ Int64 dataOffset_;
+ LobsRequest type_;
+ LobsStorage storage_;
+ Int64 operLen_;
+ Ex_Lob_Error error_;
+ Int64 cliError_;
+ Ex_Lob_Error status_;
+ Int64 transId_;
+ SB_Transid_Type transIdBig_;
+ SB_Transseq_Type transStartId_;
+ char descFileName_[MAX_LOB_FILE_NAME_LEN];
+ char blackBox_[MAX_BLACK_BOX_LEN];
+ Int64 blackBoxLen_;
+};
///////////////////////////////////////////////////////////////////////////////
// THE SQL LOB API
@@ -87,11 +160,11 @@ class ExLobGlobals;
Ex_Lob_Error ExLobsOper (
char *lobName, // lob name
char *handleIn, // input handle (for cli calls)
- Int64 handleInLen, // input handle len
+ Int32 handleInLen, // input handle len
char *hdfsServer, // server where hdfs fs resides
Int64 hdfsPort, // port number to access hdfs server
char *handleOut, // output handle (for cli calls)
- Int64 &handleOutLen, // output handle len
+ Int32 &handleOutLen, // output handle len
Int64 descNumIn, // input desc Num (for flat files only)
Int64 &descNumOut, // output desc Num (for flat files only)
Int64 &retOperLen, // length of data involved in this operation
@@ -111,9 +184,10 @@ Ex_Lob_Error ExLobsOper (
void *&globPtr, // ptr to the Lob objects.
Int64 transId,
void *blackBox, // black box to be sent to cli
- Int64 blackBoxLen, // length of black box
+ Int32 blackBoxLen, // length of black box
Int64 lobMaxSize = 0, // max size of lob.
Int64 lobMaxChunkMemSize = 0 ,//max length of intermediate mem buffer used to do i/o.
+ Int64 lobGCLimit = 0, // size at which GC must be triggered
int bufferSize =0,
short replication =0,
int blocksize=0,
@@ -160,7 +234,7 @@ class ExLobDescHeader
int getDataOffset() { return dataOffset_; }
void incDataOffset(int size) { dataOffset_ += size; }
- Ex_Lob_Error print(char *descFileName);
+
private:
@@ -200,7 +274,6 @@ class ExLobDesc
void setOffset(int offset) { dataOffset_ = offset; }
void setDataState(int dataState) { dataState_ = dataState; }
- Ex_Lob_Error print();
private:
@@ -213,79 +286,46 @@ class ExLobDesc
int nextFree_;
};
-class ExLobRequest
+class ExLobInMemoryDescChunksEntry
{
- public:
- ExLobRequest();
- ~ExLobRequest();
- Ex_Lob_Error send();
-
- void setValues(char *descFileName, Int64 descNumIn, Int64 handleInLen,
- char *handleIn, LobsStorage storage, Int64 transId,
- SB_Transid_Type transIdBig,
- SB_Transseq_Type transStartId,
- char *blackBox, Int64 blackBoxLen);
- void getValues(Int64 &descNumOut, Int64 &handleOutLen,
- char *handleOut, Ex_Lob_Error &requestStatus,
- Int64 &cliError,
- char *blackBox, Int64 &blackBoxLen);
-
- Int64 getDescNumIn() { return descNumIn_; }
- void setDescNumIn(Int64 descNum) { descNumIn_ = descNum; }
- Int64 getDescNumOut() { return descNumOut_; }
- void setDescNumOut(Int64 descNum) { descNumOut_ = descNum; }
- Int64 getDataOffset() { return dataOffset_; }
- void setDataOffset(int dataOffset) { dataOffset_ = dataOffset; }
- LobsRequest getType() { return type_; }
- void setType(LobsRequest type) { type_ = type; }
- char *getDescFileName() { return descFileName_; }
- Int64 getOperLen() { return operLen_; }
- void setOperLen(Int64 len) { operLen_ = len; }
- void log();
- void getDescOut(ExLobDesc &desc);
- void putDescIn(ExLobDesc &desc);
- ExLobDesc &getDesc() { return desc_; }
- void setError(Ex_Lob_Error err) { error_ = err; }
- Ex_Lob_Error getError() { return error_; }
- Ex_Lob_Error getStatus() { return status_; }
- char *getHandleIn() { return handleIn_; }
- Int64 getHandleInLen() { return handleInLen_; }
- char *getHandleOut() { return handleOut_; }
- void setHandleOutLen(Lng32 len) { handleOutLen_ = len; }
- void setCliError(int cliErr) { cliError_ = cliErr; }
- int getCliError() { return (int)cliError_; }
- Int64 getTransId() { return transId_; }
- SB_Transid_Type getTransIdBig() { return transIdBig_; }
- SB_Transseq_Type getTransStartId() { return transStartId_; }
- Int64 getBlackBoxLen() { return blackBoxLen_; };
- void setBlackBoxLen(Int64 len) { blackBoxLen_ = len; }
- char *getBlackBox() { return blackBox_; }
-
- void incrReqNum() { reqNum_++; }
- Int64 getReqNum() { return reqNum_; }
-
- private:
- Int64 reqNum_;
- Int64 descNumIn_;
- Int64 descNumOut_;
- char handleIn_[MAX_HANDLE_IN_LEN];
- Int64 handleInLen_;
- char handleOut_[MAX_HANDLE_OUT_LEN];
- Int64 handleOutLen_;
- Int64 dataOffset_;
- LobsRequest type_;
- LobsStorage storage_;
- Int64 operLen_;
- ExLobDesc desc_;
- Ex_Lob_Error error_;
- Int64 cliError_;
- Ex_Lob_Error status_;
- Int64 transId_;
- SB_Transid_Type transIdBig_;
- SB_Transseq_Type transStartId_;
- char descFileName_[MAX_LOB_FILE_NAME_LEN];
- char blackBox_[MAX_BLACK_BOX_LEN];
- Int64 blackBoxLen_;
+public:
+ ExLobInMemoryDescChunksEntry():
+ currentOffset_ (-1),
+ newOffset_( -1),
+ descPartnKey_(-1),
+ descSyskey_(-1),
+ chunkLen_(-1),
+ chunkNum_(-1)
+ {}
+ ExLobInMemoryDescChunksEntry(Int64 co,Int64 partnKey,Int64 syskey,Int64 chunkLen,Int32 chunkNum)
+ {
+ currentOffset_ = co; newOffset_ = -1;descPartnKey_ = partnKey; descSyskey_ = syskey; chunkLen_ = chunkLen; chunkNum_ =chunkNum;
+ }
+
+ void setCurrentOffset(Int64 co) { currentOffset_ = co;}
+ Int64 getCurrentOffset(){return currentOffset_;}
+
+ void setNewOffset(Int64 no) { newOffset_ = no;}
+ Int64 getNewOffset(){return newOffset_;}
+
+ void setDescPartnKey(Int64 dk) { descPartnKey_ = dk;}
+ Int64 getDescPartnKey(){return descPartnKey_;}
+
+ void setSyskey(Int64 sk) { descSyskey_ = sk;}
+ Int64 getSyskey(){return descSyskey_;}
+
+ void setChunkLen(Int64 cl) { chunkLen_ = cl;}
+ Int64 getChunkLen(){return chunkLen_;}
+
+ void setChunkNum(Int32 cn) { chunkNum_ = cn;}
+ Int32 getChunkNum(){return chunkNum_;}
+private:
+ Int64 currentOffset_;
+ Int64 newOffset_;
+ Int64 descPartnKey_;
+ Int64 descSyskey_;
+ Int64 chunkLen_;
+ Int32 chunkNum_;
};
class ExLobCursorBuffer
@@ -369,105 +409,103 @@ class ExLob
Ex_Lob_Error initialize(char *lobFile, Ex_Lob_Mode mode, char *dir,
LobsStorage storage, char *hdfsServer, Int64 hdfsPort,
- int bufferSize = 0, short replication =0, int blocksize=0, Int64 lobMaxSize = 0, ExLobGlobals *lobGlobals = NULL);
+ char *lobLocation,
+ int bufferSize = 0, short replication =0,
+ int blocksize=0, Int64 lobMaxSize = 0,
+ ExLobGlobals *lobGlobals = NULL);
Ex_Lob_Error initialize(char *lobFile);
- Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize);
+ Ex_Lob_Error writeDesc(Int64 &sourceLen, char *source, LobsSubOper subOperation, Int64 &descNumOut, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,Int64 lobGCLimit, char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error writeLobData(char *source, Int64 sourceLen,
LobsSubOper subOperation,
Int64 tgtOffset,Int64 &operLen,
Int64 lobMaxMemChunkLen);
Ex_Lob_Error writeDataSimple(char *data, Int64 size, LobsSubOper subOperation, Int64 &operLen,
int bufferSize = 0, short replication =0, int blocksize=0);
- Ex_Lob_Error readToMem(char *memAddr, Int64 size, Int64 &operLen);
- Ex_Lob_Error readToFile(char *fileName, Int64 tgtLen,Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int64 handleInLen, Int64 &operLen);
- Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
+ Ex_Lob_Error readToMem(char *memAddr, Int64 size, Int64 &operLen,char * handleIn, Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
+ Ex_Lob_Error readToFile(char *fileName, Int64 tgtLen,Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags, char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
+ Ex_Lob_Error readCursor(char *tgt, Int64 tgtSize, char *handleIn, Int32 handleInLen, Int64 &operLen,Int64 transId);
+ Ex_Lob_Error readCursorData(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen,char *handleIn, Int32 handeLenIn,Int64 transId);
Ex_Lob_Error readCursorDataSimple(char *tgt, Int64 tgtSize, cursor_t &cursor, Int64 &operLen);
Ex_Lob_Error readDataCursorSimple(char *fileName, char *tgt, Int64 tgtSize, Int64 &operLen, ExLobGlobals *lobGlobals);
bool hasNoOpenCursors() { return lobCursors_.empty(); }
- Ex_Lob_Error openCursor(char *handleIn, Int64 handleInLen);
+ Ex_Lob_Error openCursor(char *handleIn, Int32 handleInLen,Int64 transId);
Ex_Lob_Error openDataCursor(char *fileName, LobsCursorType type, Int64 range,
Int64 bytesLeft, Int64 bufMaxSize, Int64 prefetch, ExLobGlobals *lobGlobals);
Ex_Lob_Error deleteCursor(char *cursorName, ExLobGlobals *lobGlobals);
- Ex_Lob_Error fetchCursor();
- Ex_Lob_Error selectCursorDesc(ExLobRequest *request);
- Ex_Lob_Error fetchCursorDesc(ExLobRequest *request);
- Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize);
- Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen);
- Ex_Lob_Error append(ExLobRequest *request);
- Ex_Lob_Error update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemLen);
- Ex_Lob_Error update(ExLobRequest *request);
+ Ex_Lob_Error fetchCursor(char *handleIn, Int32 handleLenIn, Int64 &outOffset, Int64 &outSize,NABoolean &isEOD,Int64 transId);
+ Ex_Lob_Error insertData(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemSize,char *handleIn,Int32 handleInLen, char *blackBox, Int32 blackBoxLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
+ Ex_Lob_Error append(char *data, Int64 size, LobsSubOper so, Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize, Int64 lobMaxChunkMemLen,Int64 lobGCLimit, char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
+ Ex_Lob_Error update(char *data, Int64 size, LobsSubOper so,Int64 headDescNum, Int64 &operLen, Int64 lobMaxSize,Int64 lobMaxChunkMemLen,Int64 lobGCLimit,char *handleIn,Int32 handleInLen, char * handleOut, Int32 &handleOutLen, void *lobGlobals);
Ex_Lob_Error readSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readHdfsSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readLocalSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error readExternalSourceFile(char *srcfile, char *&fileData, Int32 &size, Int64 offset);
Ex_Lob_Error statSourceFile(char *srcfile, Int64 &sourceEOF);
- Ex_Lob_Error delDesc();
- Ex_Lob_Error delDesc(ExLobRequest *request);
+ Ex_Lob_Error delDesc(char *handleIn, Int32 handleInLen, Int64 transId);
Ex_Lob_Error purgeLob();
Ex_Lob_Error closeFile();
LobInputOutputFileType fileType(char *ioFileName);
- Ex_Lob_Error closeCursor(char *handleIn, Int64 handleInLen);
+ Ex_Lob_Error closeCursor(char *handleIn, Int32 handleInLen);
Ex_Lob_Error closeDataCursorSimple(char *fileName, ExLobGlobals *lobGlobals);
- Ex_Lob_Error closeCursorDesc(ExLobRequest *request);
+
Ex_Lob_Error doSanityChecks(char *dir, LobsStorage storage,
- Int64 handleInLen, Int64 handleOutLen,
- Int64 blackBoxLen);
- Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize);
- Ex_Lob_Error getNextFreeOffset(Int32 size,Int64 &freeOffset, Int64 lobMaxSize);
+ Int32 handleInLen, Int32 handleOutLen,
+ Int32 blackBoxLen);
+ Ex_Lob_Error allocateDesc(unsigned int size, Int64 &descNum, Int64 &dataOffset,Int64 lobMaxSize,Int64 lobMaxChunkMemSize, char *handleIn, Int32 handleInLen,Int64 lobGCLimit, void *lobGlobals);
Ex_Lob_Error readStats(char *buffer);
Ex_Lob_Error initStats();
- Ex_Lob_Error insertDesc(ExLobRequest *request) ;
-
- ExLobRequest *getRequest() { return &request_; }
+ Ex_Lob_Error insertDesc(Int64 offset, Int64 size, char *handleIn, Int32 handleInLen, char *handleOut, Int32 &handleOutLen, char *blackBox, Int32 blackBoxLen,void *lobGlobals) ;
Ex_Lob_Error lockDesc();
Ex_Lob_Error unlockDesc();
- //int getfdDesc() { return fdDesc_; }
- hdfsFile *getfdDesc() { return &fdDesc_;}
- char *getDescFileName() { return lobDescFile_; }
char *getDataFileName() { return lobDataFile_; }
- Ex_Lob_Error getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize);
- Ex_Lob_Error print();
+
int getErrNo();
- Ex_Lob_Error delDesc(Int64 descNum);
- Ex_Lob_Error getDesc(ExLobDesc &desc);
- Ex_Lob_Error putDesc(ExLobDesc &desc, Int64 descNum);
- Ex_Lob_Error getDesc(ExLobRequest *request);
+
+ Ex_Lob_Error getDesc(ExLobDesc &desc,char * handleIn, Int32 handleInLen, char *blackBox, Int32 *blackBoxLen, char * handleOut, Int32 &handleOutLen, Int64 transId);
Ex_Lob_Error writeData(Int64 offset, char *data, Int32 size, Int64 &operLen);
- Ex_Lob_Error readDataToMem(char *memAddr, Int64 offset, Int64 size, Int64 &operLen);
+ Ex_Lob_Error readDataToMem(char *memAddr, Int64 offset, Int64 size,
+ Int64 &operLen,char *handleIn, Int32 handleLenIn,
+ NABoolean multipleChunks, Int64 transId);
- Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 size,Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags);
- Ex_Lob_Error readCursorDataToLocalFile(char *fileName, Int64 offset, Int64 size, Int64 &writeOperLen, Int64 lobMaxChunkMemSize, Int32 fileflags);
- Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags);
- Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
+ Ex_Lob_Error readDataToLocalFile(char *fileName, Int64 offset, Int64 size,Int64 &operLen,Int64 lobMaxChunkMemLen ,Int32 fileFlags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataToHdfsFile(char *fileName, Int64 offset, Int64 size, Int64 &operLen,Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataToExternalFile(char *tgtFileName, Int64 offset, Int64 size, Int64 &operLen, Int64 lobMaxChunkMemLen, Int32 fileflags,char *handleIn,Int32 handleInLen, NABoolean multipleChunks,Int64 transId);
+ Ex_Lob_Error readDataFromFile(char *memAddr, Int64 len, Int64 &operLen);
+ Ex_Lob_Error compactLobDataFile(ExLobInMemoryDescChunksEntry *dcArray,Int32 numEntries);
+ Ex_Lob_Error restoreLobDataFile();
+ Ex_Lob_Error purgeBackupLobDataFile();
Ex_Lob_Error emptyDirectory();
ExLobStats *getStats() { return &stats_; }
NAHeap *getLobGlobalHeap() { return lobGlobalHeap_;}
+ ExLobRequest *getRequest() { return &request_; }
+
+ //The next 2 functions are not active at this point. They serve as an example
+ //on how to send requests across to the mxlobsrvr process from the master
+ //process
+ Ex_Lob_Error getDesc(ExLobRequest *request);
+ Ex_Lob_Error sendReqToLobServer() ;
public:
char lobDataFile_[MAX_LOB_FILE_NAME_LEN];
- char lobDescFile_[MAX_LOB_FILE_NAME_LEN];
- //int fdDesc_;
- hdfsFile fdDesc_;
lobCursors_t lobCursors_;
ExLobLock lobCursorLock_;
LobsStorage storage_;
string dir_; // lob data directory
char *hdfsServer_;
Int64 hdfsPort_;
+ char *lobLocation_;
hdfsFS fs_;
hdfsFile fdData_;
int openFlags_;
- ExLobRequest request_;
ExLobStats stats_;
bool prefetchQueued_;
NAHeap *lobGlobalHeap_;
+ ExLobRequest request_;
};
typedef map<string, ExLob *> lobMap_t;
@@ -537,8 +575,6 @@ class ExLobGlobals
~ExLobGlobals();
Ex_Lob_Error initialize();
- Ex_Lob_Error setServerPhandle();
- Ex_Lob_Error resetServerPhandle();
lobMap_t * getLobMap() { return lobMap_; }
Ex_Lob_Error getLobPtr(char *lobName, ExLob *& lobPtr);
Ex_Lob_Error delLobPtr(char *lobName);
@@ -598,4 +634,6 @@ class ExLobGlobals
NAHeap *heap_;
};
+
+
#endif
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBenums.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBenums.h b/core/sql/exp/ExpLOBenums.h
index 67f5f43..99e072d 100644
--- a/core/sql/exp/ExpLOBenums.h
+++ b/core/sql/exp/ExpLOBenums.h
@@ -237,7 +237,10 @@ typedef enum {
Lob_Empty_Directory,
- Lob_Cleanup // destroy everything under globals
+ Lob_Cleanup, // destroy everything under globals
+ Lob_PerformGC,
+ Lob_RestoreLobDataFile,
+ Lob_PurgeBackupLobDataFile
} LobsOper;
typedef enum {
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBexternal.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBexternal.h b/core/sql/exp/ExpLOBexternal.h
index b21b97d..e8080f6 100644
--- a/core/sql/exp/ExpLOBexternal.h
+++ b/core/sql/exp/ExpLOBexternal.h
@@ -93,7 +93,11 @@ enum LOBcliQueryType
LOB_DATA_EXTRACT,
// returns length of lob given a lobhandle
- LOB_CLI_SELECT_LOBLENGTH
+ LOB_CLI_SELECT_LOBLENGTH,
+
+ // performs GC of lob file
+ LOB_CLI_PERFORM_LOB_GC
+
};
enum LOBcliQueryPhase
@@ -116,7 +120,7 @@ Lng32 SQL_EXEC_LOBcliInterface
/*IN*/ char * inLobHandle,
/*IN*/ Lng32 inLobHandleLen,
/*IN*/ char * blackBox,
- /*IN*/ Lng32* blackBoxLen,
+ /*IN*/ Int32* blackBoxLen,
/*OUT*/ char * outLobHandle,
/*OUT*/ Lng32 * outLobHandleLen,
/*IN*/ LOBcliQueryType qType,
@@ -135,6 +139,16 @@ Lng32 SQL_EXEC_LOBcliInterface
in */
/*IN*/ Int64 xnId /* xn id of the parent process, if non-zero */
);
+Lng32 SQL_EXEC_LOB_GC_Interface
+(
+ /*IN*/ void *lobGlobals, // can be passed or NULL
+ /*IN*/ char * handle,
+ /*IN*/ Lng32 handleLen,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
+ /*IN*/ char *lobLocation,
+ /*IN*/ Int64 lobMaxMemChunkLen // if passed in as 0, will use default value of 1G for the in memory buffer to do compaction.
+ );
Lng32 SQL_EXEC_LOBddlInterface
(
@@ -146,6 +160,8 @@ Lng32 SQL_EXEC_LOBddlInterface
/*IN*/ short *lobNumList,
/*IN*/ short *lobTypList,
/*IN*/ char* *lobLocList,
+ /*IN*/ char* hdfsServer,
+ /*IN*/ Lng32 hdfsPort,
/*IN*/ Int64 lobMaxSize
);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBinterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.cpp b/core/sql/exp/ExpLOBinterface.cpp
index d55bab9..188fed4 100644
--- a/core/sql/exp/ExpLOBinterface.cpp
+++ b/core/sql/exp/ExpLOBinterface.cpp
@@ -41,13 +41,13 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int6
Int64 dummyParam = 0;
Int64 cliError = -1;
Ex_Lob_Error status;
-
+ Int32 dummyParam2 = 0;
char dir[100];
- strcpy(dir, "/h/temp");
+ strcpy(dir, "/lobs");
err = ExLobsOper(dir,
NULL, 0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
dir, Lob_HDFS_File,
NULL, 0,
@@ -76,22 +76,101 @@ Lng32 ExpLOBinterfaceInit(void *& lobGlob, void * lobHeap,NABoolean isHive, Int6
return 0;
}
-Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap)
+Lng32 ExpLOBinterfacePerformGC(void *& lobGlob, char *lobName,void *descChunksArray, Int32 numEntries, char *hdfsServer, Int32 hdfsPort,char *lobLoc,Int64 lobMaxChunkMemSize)
{
Ex_Lob_Error err;
+ Ex_Lob_Error status;
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ (char *)descChunksArray, numEntries,
+ 0,NULL,
+ Lob_PerformGC, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfaceRestoreLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName)
+{
+ Ex_Lob_Error err;
+ Ex_Lob_Error status;
Int64 dummyParam = 0;
Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ NULL, 0,
+ 0,NULL,
+ Lob_RestoreLobDataFile, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName)
+{
+ Ex_Lob_Error err;
Ex_Lob_Error status;
-
- char dir[100];
- strcpy(dir, "/h/temp");
- err = ExLobsOper(dir,
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper(lobName,
+ NULL, 0,
+ hdfsServer, hdfsPort,
+ NULL, dummyParam2, 0, dummyParam,
+ dummyParam, 0, dummyParam, status, cliError,
+ lobLoc, Lob_HDFS_File,
+ NULL, 0,
+ 0,NULL,
+ Lob_PurgeBackupLobDataFile, // Lob_GC
+ Lob_None,
+ 1, // waited op
+ lobGlob,
+ 0,
+ NULL, 0
+ );
+ if (err != LOB_OPER_OK)
+ return -1;
+ else
+ return 0;
+}
+Lng32 ExpLOBinterfaceCleanup(void *& lobGlob, void * lobHeap)
+{
+ Ex_Lob_Error err;
+ Ex_Lob_Error status;
+ Int64 dummyParam = 0;
+ Int64 cliError = -1;
+ Int32 dummyParam2 = 0;
+ err = ExLobsOper((char *)"dummy",
NULL, 0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
- dir, Lob_HDFS_File,
+ NULL, Lob_HDFS_File,
NULL, 0,
0,NULL,
Lob_Cleanup, // Lob_Cleanup
@@ -119,14 +198,15 @@ Lng32 ExpLOBinterfaceCreate(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType, //Lob_HDFS_File,
NULL, 0,
@@ -148,17 +228,7 @@ Lng32 ExpLOBinterfaceCreate(
else
return 0;
}
-/*
- * Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob,
- char * lobName,
- char * lobLoc,
- Lng32 lobType = (Lng32)Lob_Empty_Directory,
- char * lobHdfsServer = NULL,
- Lng32 lobHdfsPort = 0,
- int bufferSize = 0,
- short replication =0,
- int blocksize=0);
- */
+
Lng32 ExpLOBinterfaceEmptyDirectory(
void * lobGlob,
char * lobName,
@@ -172,14 +242,15 @@ Lng32 ExpLOBinterfaceEmptyDirectory(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType, //Lob_HDFS_File,
NULL, 0,
@@ -204,14 +275,15 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob, char * lobHdfsServer ,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, Lob_HDFS_File,
NULL, 0,
@@ -230,21 +302,20 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob, char * lobHdfsServer ,
}
Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
- char * lobHdfsServer ,
- Lng32 lobHdfsPort ,char * lobName, char * lobLoc)
+ char * lobName, char * lobLoc)
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
- // char dir[100];
- // strcpy(dir, "/h/vshah");
+
err = ExLobsOper(lobName,
- lobHdfsServer, lobHdfsPort,
+ NULL,0,
NULL, 0,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, Lob_HDFS_File,
NULL, 0,
@@ -271,7 +342,8 @@ Lng32 ExpLOBinterfaceCloseFile(void * lobGlob,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
@@ -280,7 +352,7 @@ Lng32 ExpLOBinterfaceCloseFile(void * lobGlob,
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, ls, //Lob_HDFS_File,
NULL, 0,
@@ -309,7 +381,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 blackBoxLen,
@@ -329,6 +401,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Int64 srcLobLen,
Int64 lobMaxSize,
Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit,
int bufferSize ,
short replication ,
int blockSize)
@@ -373,22 +446,14 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
blackBox, blackBoxLen,
lobMaxSize,
lobMaxChunkMemSize,
+ lobGCLimit,
bufferSize,
replication,
blockSize
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
-
- // done with the request (success or error)
- requestTag = -1;
+
if (err != LOB_OPER_OK)
{
if ((cliError) &&
@@ -399,14 +464,9 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
}
descSyskey = outDescSyskey;
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
-
- // error. Status contains the error code.
- return -(short)status;
+
+
+ return 0;
}
Lng32 ExpLOBInterfaceInsertSelect(void * lobGlob,
@@ -434,13 +494,14 @@ Lng32 ExpLOBInterfaceInsertSelect(void * lobGlob,
LobsSubOper lso = Lob_Foreign_Lob;
Ex_Lob_Error status;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Int64 cliError = -1;
err = ExLobsOper(tgtLobName,
lobHandle, handleLen,
NULL, 0, // hdfs server/port
- NULL, dummyParam, -1, descSyskey, lobLen,
+ NULL, dummyParam2, -1, descSyskey, lobLen,
0, dummyParam, status, cliError,
lobStorageLocation, Lob_HDFS_File,
lobData, lobLen, //strlen(srcLobData),
@@ -465,7 +526,7 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobStorageLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -482,7 +543,8 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize,
- Int64 lobMaxChunkMemSize
+ Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit
)
{
Ex_Lob_Error err;
@@ -512,32 +574,19 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
lobGlob,
xnId, NULL, 0,
lobMaxSize,
- lobMaxChunkMemSize
+ lobMaxChunkMemSize,
+ lobGCLimit
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
+
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+ return 0;
}
Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
@@ -548,7 +597,7 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
char * lobStorageLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -565,7 +614,8 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize ,
- Int64 lobMaxChunkMemSize )
+ Int64 lobMaxChunkMemSize,
+ Int64 lobGCLimit)
{
Ex_Lob_Error err;
@@ -596,32 +646,20 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
xnId,
NULL, 0,
lobMaxSize,
- lobMaxChunkMemSize
+ lobMaxChunkMemSize,
+ lobGCLimit
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
+
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+
+ return 0;
}
Lng32 ExpLOBInterfaceDelete(void * lobGlob,
@@ -639,7 +677,8 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
@@ -655,7 +694,7 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort, // hdfs server/port
- NULL, dummyParam,
+ NULL, dummyParam2,
descSyskey, dummyParam,
dummyParam,
requestTag, requestTag,
@@ -671,29 +710,14 @@ Lng32 ExpLOBInterfaceDelete(void * lobGlob,
NULL, 0
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
- // done with the request (success or error)
- requestTag = -1;
if (err != LOB_OPER_OK)
{
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
- // error. Status contains the error code.
- return -(short)status;
+ return 0;
}
@@ -719,15 +743,14 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
)
{
Ex_Lob_Error err;
-
- Int64 dummyParam;
Ex_Lob_Error status;
+ Int64 dummyParam=0;
+ Int32 dummyParam2 = 0;
Int64 cliError;
LobsOper lo;
- if (checkStatus)
- lo = Lob_Check_Status;
- else if (lobHandle == NULL)
+
+ if (lobHandle == NULL)
{
requestTag = -1;
lo = Lob_ReadDataSimple;
@@ -743,7 +766,7 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam,
+ NULL, dummyParam2,
srcOffset, dummyParam,
outLen,
requestTag, requestTag,
@@ -759,17 +782,10 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
NULL, 0,
0,
lobMaxMemChunkLen,
- 0,0,0,inputFlags
+ 0,0,0,0,inputFlags
);
- if ((err == LOB_OPER_OK) &&
- (status == LOB_OPER_REQ_IN_PROGRESS))
- {
- // this request has been sent and is in progress.
- // requestTag contains the id of the request.
- return LOB_ACCESS_PREEMPT;
- }
-
+
// done with the request (success or error)
requestTag = -1;
if (err != LOB_OPER_OK)
@@ -777,14 +793,9 @@ Lng32 ExpLOBInterfaceSelect(void * lobGlob,
return -(short)err;
}
- if ((waitedOp) ||
- ((Ex_Lob_Error)status == LOB_OPER_OK))
- {
- return LOB_ACCESS_SUCCESS;
- }
// error. Status contains the error code.
- return -(short)status;
+ return 0;
}
Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
@@ -794,7 +805,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
char * lobHdfsServer,
Lng32 lobHdfsPort,
- Int64 handleLen,
+ Int32 handleLen,
char * lobHandle,
Int64 cursorBytes,
@@ -815,6 +826,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
Ex_Lob_Error err;
Int64 dummyParam = 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError;
@@ -853,7 +865,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
err = ExLobsOper(lobName,
lobHandle, handleLen,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam,
+ NULL, dummyParam2,
srcOffset, dummyParam,
outLen,
requestTag, requestTag,
@@ -866,7 +878,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
waitedOp,
lobGlob,
0,
- NULL, 0,0,0,0,0,0,
+ NULL, 0,0,0,0,0,0,0,
openType
);
@@ -888,14 +900,15 @@ Lng32 ExpLOBinterfaceStats(
{
Ex_Lob_Error err;
- Int64 dummyParam;
+ Int64 dummyParam= 0;
+ Int32 dummyParam2 = 0;
Ex_Lob_Error status;
Int64 cliError = -1;
err = ExLobsOper(lobName,
NULL, 0,
lobHdfsServer, lobHdfsPort,
- NULL, dummyParam, 0, dummyParam,
+ NULL, dummyParam2, 0, dummyParam,
dummyParam, 0, dummyParam, status, cliError,
lobLoc, (LobsStorage)lobType,
(char*)lobStats, 0,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBinterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBinterface.h b/core/sql/exp/ExpLOBinterface.h
index 646d846..26f2ad4 100644
--- a/core/sql/exp/ExpLOBinterface.h
+++ b/core/sql/exp/ExpLOBinterface.h
@@ -98,9 +98,7 @@ Lng32 ExpLOBinterfaceDrop(void * lobGlob,
char * lobName,
char * lobLoc);
-Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
- char * lobHdfsServer ,
- Lng32 lobHdfsPort ,
+Lng32 ExpLOBInterfacePurgedata(void * lobGlob,
char * lobName,
char * lobLoc);
@@ -135,7 +133,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Lng32 handleLen,
char * lobHandle,
- Int64 * outHandleLen,
+ Int32 * outHandleLen,
char * outLobHandle,
Int64 blackBoxLen,
@@ -156,6 +154,7 @@ Lng32 ExpLOBInterfaceInsert(void * lobGlob,
Int64 srcLobLen = 0,
Int64 lobMaxSize = 0,
Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0,
int bufferSize = 0,
short replication =0,
int blocksize=0
@@ -168,7 +167,7 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
char * lobLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -188,7 +187,8 @@ Lng32 ExpLOBInterfaceUpdate(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize = 0,
- Int64 lobMaxChunkMemSize = 0);
+ Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0);
Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobHdfsServer ,
@@ -197,7 +197,7 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
char * lobLocation,
Lng32 handleLen,
char * lobHandle,
- Int64 *outHandleLen,
+ Int32 *outHandleLen,
char * outLobHandle,
Int64 &requestTag,
Int64 xnId,
@@ -217,7 +217,8 @@ Lng32 ExpLOBInterfaceUpdateAppend(void * lobGlob,
Int64 srcDescKey,
Int64 srcDescTS,
Int64 lobMaxSize = 0,
- Int64 lobMaxChunkMemSize = 0
+ Int64 lobMaxChunkMemSize = 0,
+ Int64 lobGCLimit = 0
);
Lng32 ExpLOBInterfaceDelete(void * lobGlob,
@@ -260,7 +261,7 @@ Lng32 ExpLOBInterfaceSelectCursor(void * lobGlob,
char * lobHdfsServer,
Lng32 lobHdfsPort,
- Int64 handleLen,
+ Int32 handleLen,
char * lobHandle,
Int64 cusrorBytes,
char *cursorId,
@@ -286,6 +287,11 @@ Lng32 ExpLOBinterfaceStats(void * lobGlob,
char * getLobErrStr(Lng32 errEnum);
+Lng32 ExpLOBinterfacePerformGC(void *& lobGlob, char *lobName,void *descChunksArray, Int32 numEntries, char *hdfsServer, Int32 hdfsPort,char *LOBlOC,Int64 lobMaxChunkMemSize);
+Lng32 ExpLOBinterfaceRestoreLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName);
+Lng32 ExpLOBinterfacePurgeBackupLobDataFile(void *& lobGlob, char *hdfsServer, Int32 hdfsPort,char *lobLoc,char *lobName);
+
+
Lng32 ExpLOBinterfaceEmptyDirectory(void * lobGlob,
char * lobName,
char * lobLoc,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBprocess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.cpp b/core/sql/exp/ExpLOBprocess.cpp
index e121c1e..e520c4c 100644
--- a/core/sql/exp/ExpLOBprocess.cpp
+++ b/core/sql/exp/ExpLOBprocess.cpp
@@ -23,8 +23,8 @@
/* -*-C++-*-
*****************************************************************************
*
- * File: ex_lob.C
- * Description: class to store and retrieve LOB data.
+ * File: ExpLOBprocess.cpp
+ * Description: class to store and retrieve LOB info from mxlobsrvr process.
*
*
* Created: 10/29/2012
@@ -35,7 +35,13 @@
*
*****************************************************************************
*/
+/*** Note *** This file is currently compiled and creates the mxlobsrvr executable. But the functions in this file are not active or used at this point. Code
+ maybe added in the near future to offload any tasks like garbage collection, to this process .Hence we are retainign this file as part of the mxlobsrvr
+infrastructure .If any functionas are added and need to be executed in the
+mxlobsrvr process, the sqstart/sqstop need to modified to call lobstop and
+lostart**/
+/****************************************************************************/
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
@@ -351,85 +357,7 @@ void process_mon_msg(MS_Mon_Msg *msg) {
}
}
-Ex_Lob_Error ExLob::append(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- Int64 size = request->getDesc().getSize();
- offset = request->getDataOffset();
-
- /* This is done in the master
- // allocate a new lob desc
- err = allocateDesc(size, dummyParam, offset);
-
- if (err != LOB_OPER_OK)
- return err;
-
- request->getDesc().setOffset(offset);
- */
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_INSERT_APPEND, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::update(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- Int64 size = request->getDesc().getSize();
- offset = request->getDataOffset();
- /* sss this is done in the master
- // allocate a new lob desc
- err = allocateDesc(size, dummyParam, offset);
-
- if (err != LOB_OPER_OK)
- return err;
-
- request->getDesc().setOffset(offset);
- */
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_UPDATE_UNIQUE, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-Ex_Lob_Error ExLob::putDesc(ExLobDesc &desc, Int64 descNum)
-{
- /* Ex_Lob_Error err = LOB_OPER_OK;
-
- Lng32 offset = sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * descNum;
- if (pwrite(fdDesc_, &desc, sizeof(ExLobDesc), offset) == -1) {
- return LOB_DESC_WRITE_ERROR;
- }
- */
- return LOB_OPER_OK;
-}
Ex_Lob_Error ExLob::getDesc(ExLobRequest *request)
{
@@ -452,255 +380,14 @@ Ex_Lob_Error ExLob::getDesc(ExLobRequest *request)
request->setHandleOutLen(handleOutLen);
request->setBlackBoxLen(blackBoxLen);
- request->getDesc().setOffset(offset);
- request->getDesc().setSize(size);
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::delDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 offset;
- Int64 dummyParam;
- Lng32 handleOutLen;
- Lng32 clierr;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_DELETE, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-/*Ex_Lob_Error ExLob::purgeLob()
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
- close(fdDesc_);
- fdDesc_ = -1;
-
- if (remove(lobDescFile_) != 0) {
- return LOB_DESC_FILE_DELETE_ERROR;
- }
-
- return LOB_OPER_OK;
- } */
-
-Ex_Lob_Error ExLob::purgeLob()
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
- hdfsCloseFile(fs_,fdDesc_);
- fdDesc_ = NULL;
-
- if (hdfsDelete(fs_,lobDescFile_,0) != 0) {
- return LOB_DESC_FILE_DELETE_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-
-
-Ex_Lob_Error ExLob::getNextFreeOffset(Int64 size, Int64 &dataOffset, Int64 lobMaxSize)
-{
- dataOffset = 0;
- // Retrieve the next available, best-fit free offset to write into.
-
- if (sourceLen <= 0 || sourceLen > lobMaxSize)
- {
- return LOB_MAX_LIMIT_ERROR; //exceeded the size of the max lob size
- }
-}
-
-
-/*Ex_Lob_Error ExLob::allocateDesc(ULng32 size, Int64 &descNum, Int64 &dataOffset)
-{
- Ex_Lob_Error err = LOB_OPER_OK;
-
-
- err = lockDesc();
- if (err != LOB_OPER_OK)
- return err;
- ExLobDescHeader header;
-
- if (pread(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- flock(fdDesc_, LOCK_UN);
- return LOB_DESC_HEADER_READ_ERROR;
- }
-
- if (header.getAvailSize() >= size) {
- descNum = header.getFreeDesc();
- dataOffset = header.getDataOffset();
- header.incFreeDesc();
- header.decAvailSize(size);
- header.incDataOffset(size);
-
- if (pwrite(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- err = LOB_DESC_HEADER_WRITE_ERROR;
- }
-
- }
- else {
- err = LOB_DATA_FILE_FULL_ERROR;
- }
-
-
- ExLobDesc desc(dataOffset, size, descNum);
- if (pwrite(fdDesc_, &desc, sizeof(ExLobDesc),
- sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * descNum) == -1) {
- return LOB_DESC_WRITE_ERROR;
- }
-
- err = unlockDesc();
- if (err != LOB_OPER_OK)
- return err;
-
- return err;
-} */
-
-
-
-Ex_Lob_Error ExLob::insertDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Lng32 clierr;
- Int64 dummyParam;
- Lng32 handleOutLen = 0;
- Lng32 blackBoxLen = 0;
- Int64 offset = request->getDataOffset();
- Int64 size = request->getDesc().getSize();
-
- blackBoxLen = request->getBlackBoxLen();
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- request->getBlackBox(), &blackBoxLen,
- request->getHandleOut(), &handleOutLen,
- LOB_CLI_INSERT, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- 0,
- request->getTransId());
-
- request->setHandleOutLen(handleOutLen);
- request->setBlackBoxLen(blackBoxLen);
request->setCliError(clierr);
return LOB_OPER_OK;
}
-Ex_Lob_Error ExLob::selectCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- void *cliInterface = NULL;
- cursor_t cursor;
- int clierr;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_CURSOR, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- cursor.cliInterface_ = cliInterface;
- // these are only used in library, but initialize them here anyway.
- cursor.bytesRead_ = -1;
- cursor.descOffset_ = -1;
- cursor.descSize_ = -1;
- cursor.eod_ = false;
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
- if (it == lobCursors_.end())
- {
- lobCursors_.insert(pair<string, cursor_t>
- (string(request->getHandleIn(), request->getHandleInLen()), cursor));
- }
- else
- {
- it->second = cursor;
- }
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::fetchCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- Int64 offset = -1;
- Int64 size = -1;
- int clierr;
-
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
-
- if (it == lobCursors_.end())
- {
- return LOB_CURSOR_NOT_OPEN;
- }
-
- void *cliInterface = it->second.cliInterface_;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- 0, 0,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_FETCH, LOB_CLI_ExecImmed,
- &offset, &size,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- request->getDesc().setOffset(offset);
- request->getDesc().setSize(size);
- request->setCliError(clierr);
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::closeCursorDesc(ExLobRequest *request)
-{
- Ex_Lob_Error err;
- Int64 dummyParam;
- int clierr;
-
- lobCursors_it it = lobCursors_.find(string(request->getHandleIn(), request->getHandleInLen()));
-
- if (it == lobCursors_.end())
- {
- return LOB_CURSOR_NOT_OPEN;
- }
-
- void *cliInterface = it->second.cliInterface_;
-
- clierr = SQL_EXEC_LOBcliInterface(request->getHandleIn(), request->getHandleInLen(),
- NULL, NULL,
- (char *)&dummyParam, (Lng32 *)&dummyParam,
- LOB_CLI_SELECT_CLOSE, LOB_CLI_ExecImmed,
- &dummyParam, &dummyParam,
- &dummyParam, &dummyParam,
- &cliInterface,
- request->getTransId());
-
- // do not update cliErr in request because the fetch would have updated it.
-
- // remove cursor from the map.
- err = closeCursor(request->getHandleIn(), request->getHandleInLen());
-
- return LOB_OPER_OK;
-}
void processRequest(ExLobRequest *request)
{
@@ -712,7 +399,7 @@ void processRequest(ExLobRequest *request)
ExLob *lobPtr;
ExLobDesc desc;
ExLobDesc *descPtr;
- Lng32 clierr;\
+ Lng32 clierr;
Lng32 handleOutLen;
Int64 size;
@@ -741,58 +428,10 @@ void processRequest(ExLobRequest *request)
}
switch(request->getType())
{
- case Lob_Req_Allocate_Desc:
- /* sss no need for this since it's done in master
- err = lobPtr->allocateDesc(request->getDesc().getSize(), descNum, dataOffset);
-
- request->setDescNumOut(descNum);
- request->setDataOffset(dataOffset);
- */
- err = lobPtr->insertDesc(request);
- break;
-
- case Lob_Req_Append:
- err = lobPtr->append(request);
- break;
-
- case Lob_Req_Select_Cursor:
- err = lobPtr->selectCursorDesc(request);
- break;
-
- case Lob_Req_Fetch_Cursor:
- err = lobPtr->fetchCursorDesc(request);
- if (request->getCliError() == 100) {
- lobPtr->closeCursorDesc(request);
- }
- break;
-
- case Lob_Req_Update:
- err = lobPtr->update(request);
- break;
-
case Lob_Req_Get_Desc:
err = lobPtr->getDesc(request);
break;
- case Lob_Req_Put_Desc:
- err = lobPtr->putDesc(request->getDesc(), request->getDescNumIn());
- break;
-
- case Lob_Req_Del_Desc:
- err = lobPtr->delDesc(request);
- break;
-
- case Lob_Req_Purge_Desc:
- /* no op ..it is dropped in mastererr = lobPtr->purgeLob();
- if (err == LOB_OPER_OK) {
- err = lobGlobals->delLobPtr(request->getDescFileName());
- } */
- break;
-
- case Lob_Req_Print:
- lobPtr->print();
- break;
-
default:
err = LOB_REQUEST_UNDEFINED_ERROR;
printf("bad request = %d\n", request->getType());
@@ -801,23 +440,12 @@ void processRequest(ExLobRequest *request)
request->setError(err);
- request->log();
+
return;
}
-void ExLobRequest::log()
-{
- printf("req num = %ld\n", getReqNum());
- printf("req type = %d\n", getType());
- printf("trans id = %ld\n", getTransId());
- printf("file = %s\n", getDescFileName());
- printf("size = %d\n", getDesc().getSize());
- printf("offset = %d\n", getDesc().getOffset());
- printf("lob err = %d\n", getError());
- printf("cli err = %d\n", getCliError());
- printf("\n");
-}
+
void receive_message(ExLobRequest *request)
{
@@ -859,27 +487,9 @@ Ex_Lob_Error ExLobGlobals::initialize()
return LOB_OPER_OK;
}
-/*Ex_Lob_Error ExLob::initialize(char *lobFile)
-{
- mode_t filePerms = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH;
- Lng32 openFlags = O_RDWR;
-
- strcpy(lobDescFile_, lobFile);
- fdDesc_ = open(lobDescFile_, openFlags);
- if (fdDesc_ == -1) {
- return LOB_DESC_FILE_OPEN_ERROR;
- }
- return LOB_OPER_OK;
- }*/
-Ex_Lob_Error ExLob::initialize(char *lobDescFile)
-{
- // Made this obsolete
-
- return LOB_OPER_OK;
-}
Ex_Lob_Error ExLobGlobals::getLobPtr(char *lobName, ExLob *& lobPtr)
{
@@ -922,106 +532,11 @@ Ex_Lob_Error ExLobGlobals::delLobPtr(char *lobName)
return LOB_OPER_OK;
}
-/*
-Ex_Lob_Error ExLob::lockDesc()
-{
-
- // get exclusive lock because read and write has to be atomic
- if (flock(fdDesc_, LOCK_EX) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-
-Ex_Lob_Error ExLob::unlockDesc()
-{
- // release the lock
-
- if (flock(fdDesc_, LOCK_UN) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- return LOB_OPER_OK;
-}
-*/
-
-
-Ex_Lob_Error ExLobDescHeader::print(char *descFileName)
-{
- printf("Header: \n");
- printf("freeDesc = %d\n", freeDesc_);
- printf("dataOffset = %d\n", dataOffset_);
- printf("avail Size = %d\n", availSize_);
- return LOB_OPER_OK;
-}
-/*Ex_Lob_Error ExLob::print()
-{
- // get exclusive lock because read and write has to be atomic
- if (flock(fdDesc_, LOCK_EX) == -1) {
- return LOB_DESC_FILE_LOCK_ERROR;
- }
-
- ExLobDescHeader header;
- if (pread(fdDesc_, &header, sizeof(ExLobDescHeader), 0) == -1) {
- return LOB_DESC_HEADER_READ_ERROR;
- }
- header.print(NULL);
- printf("Descriptors:\n");
- printf("dNum size state tail prev next nextfree offset\n");
- ExLobDesc desc;
- for (Lng32 numDescs = 0; numDescs < header.getFreeDesc(); numDescs++) {
- if (pread(fdDesc_, &desc, sizeof(ExLobDesc),
- sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * numDescs) == -1) {
- flock(fdDesc_, LOCK_UN);
- return LOB_DESC_READ_ERROR;
- }
- printf("%4d ", numDescs);
- desc.print();
- }
- flock(fdDesc_, LOCK_UN);
- return LOB_OPER_OK;
-}
-*/
-
-Ex_Lob_Error ExLob::print()
-{
- // get exclusive lock because read and write has to be atomic
- Int32 retval = 0;
- //get a mutex to prevent multiple updates
- retval = pthread_mutex_lock(&g_mutex);
- if (retval != 0 )
- return LOB_ALLOC_ERROR;
- long openFlags = O_RDONLY ;
- ExLobDescHeader header;
- if (hdfsPread(fs_,fdDesc_,0, &header, sizeof(ExLobDescHeader)) <= 0) {
- return LOB_DESC_HEADER_READ_ERROR;
- }
-
- header.print(NULL);
-
- printf("Descriptors:\n");
-
- printf("dNum size state tail prev next nextfree offset\n");
- ExLobDesc desc;
- for (Lng32 numDescs = 0; numDescs < header.getFreeDesc(); numDescs++) {
- if (hdfsPread(fs_,fdDesc_, sizeof(ExLobDescHeader) + sizeof(ExLobDesc) * numDescs, &desc, sizeof(ExLobDesc) ) <= 0 ) {
- pthread_mutex_unlock(&g_mutex);
- return LOB_DESC_READ_ERROR;
- }
- printf("%4d ", numDescs);
- desc.print();
- }
-
-
- pthread_mutex_unlock(&g_mutex);
- return LOB_OPER_OK;
-}
Lng32 main(Lng32 argc, char *argv[])
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/exp/ExpLOBprocess.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpLOBprocess.h b/core/sql/exp/ExpLOBprocess.h
index b7112d5..2e7ffb8 100644
--- a/core/sql/exp/ExpLOBprocess.h
+++ b/core/sql/exp/ExpLOBprocess.h
@@ -23,7 +23,7 @@
/* -*-C++-*-
*****************************************************************************
*
- * File: ex_lob.h
+ * File: ExLOBprocess.h
* Description: header file for LOB storage and retrieval
*
*
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/generator/GenItemFunc.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenItemFunc.cpp b/core/sql/generator/GenItemFunc.cpp
index c7e6748..a41c45a 100644
--- a/core/sql/generator/GenItemFunc.cpp
+++ b/core/sql/generator/GenItemFunc.cpp
@@ -2839,6 +2839,9 @@ short LOBinsert::codeGen(Generator * generator)
li->setLobStorageLocation((char*)lobStorageLocation().data());
li->setLobMaxSize(getLobMaxSize());
li->setLobMaxChunkMemSize(getLobMaxChunkMemSize());
+ li->setLobGCLimit(getLobGCLimit());
+ li->setLobHdfsServer((char *)getLobHdfsServer().data());
+ li->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, li);
return 0;
@@ -2864,7 +2867,8 @@ short LOBdelete::codeGen(Generator * generator)
ld->lobNum() = lobNum();
ld->setLobStorageType(lobStorageType());
ld->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ ld->setLobHdfsServer((char *)getLobHdfsServer().data());
+ ld->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, ld);
return 0;
@@ -2908,6 +2912,9 @@ short LOBupdate::codeGen(Generator * generator)
lu->setLobStorageLocation((char*)lobStorageLocation().data());
lu->setLobMaxSize(getLobMaxSize());
lu->setLobMaxChunkMemSize(getLobMaxChunkMemSize());
+ lu->setLobGCLimit(getLobGCLimit());
+ lu->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lu->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, lu);
return 0;
@@ -2929,7 +2936,8 @@ short LOBselect::codeGen(Generator * generator)
ls->lobNum() = lobNum();
ls->setLobStorageType(lobStorageType());
ls->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ ls->setLobHdfsServer((char *)getLobHdfsServer().data());
+ ls->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, ls);
return 0;
@@ -2957,7 +2965,8 @@ short LOBconvertHandle::codeGen(Generator * generator)
lu->lobNum() = lobNum();
lu->setLobStorageType(lobStorageType());
lu->setLobStorageLocation((char*)lobStorageLocation().data());
-
+ lu->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lu->setLobHdfsPort(getLobHdfsPort());
generator->getExpGenerator()->linkClause(this, lu);
return 0;
@@ -2987,6 +2996,8 @@ short LOBconvert::codeGen(Generator * generator)
lc->setLobStorageLocation((char*)lobStorageLocation().data());
generator->getExpGenerator()->linkClause(this, lc);
lc->setConvertSize(getTgtSize());
+ lc->setLobHdfsServer((char *)getLobHdfsServer().data());
+ lc->setLobHdfsPort(getLobHdfsPort());
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/optimizer/ItemFunc.h
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/ItemFunc.h b/core/sql/optimizer/ItemFunc.h
index 3bb7238..29776f0 100644
--- a/core/sql/optimizer/ItemFunc.h
+++ b/core/sql/optimizer/ItemFunc.h
@@ -2708,8 +2708,12 @@ public:
lobNum_(-1),
lobStorageType_(Lob_Invalid_Storage),
lobMaxSize_(CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)),
- lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE))
- {}
+ lobMaxChunkMemSize_(CmpCommon::getDefaultNumeric(LOB_MAX_CHUNK_MEM_SIZE)),
+ lobGCLimit_(CmpCommon::getDefaultNumeric(LOB_GC_LIMIT_SIZE)),
+ hdfsPort_((Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT)),
+ hdfsServer_( CmpCommon::getDefaultString(LOB_HDFS_SERVER))
+ {
+ }
// copyTopNode method
virtual ItemExpr * copyTopNode(ItemExpr *derivedNode = NULL,
@@ -2736,8 +2740,10 @@ public:
LobsStorage &lobStorageType() { return lobStorageType_; }
NAString &lobStorageLocation() { return lobStorageLocation_; }
Int64 getLobMaxSize() {return lobMaxSize_*1024*1024; }
- Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_;}
-
+ Int64 getLobMaxChunkMemSize() { return lobMaxChunkMemSize_*1024*1024;}
+ Int64 getLobGCLimit() { return lobGCLimit_*1025*1024;}
+ Int32 getLobHdfsPort() { return hdfsPort_;}
+ NAString &getLobHdfsServer(){return hdfsServer_;}
protected:
ObjectType obj_;
@@ -2746,6 +2752,9 @@ public:
NAString lobStorageLocation_;
Int32 lobMaxSize_; // In MB units
Int32 lobMaxChunkMemSize_; //In MB Units
+ Int32 lobGCLimit_ ;//In MB Units
+ Int32 hdfsPort_;
+ NAString hdfsServer_;
}; // LOBoper
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/optimizer/NATable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/optimizer/NATable.cpp b/core/sql/optimizer/NATable.cpp
index 3e21c79..ef46a4d 100644
--- a/core/sql/optimizer/NATable.cpp
+++ b/core/sql/optimizer/NATable.cpp
@@ -5506,7 +5506,8 @@ NATable::NATable(BindWA *bindWA,
if(postCreateNATableWarnings != preCreateNATableWarnings)
tableConstructionHadWarnings_=TRUE;
-
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
if (hasLobColumn())
{
// read lob related information from lob metadata
@@ -5546,7 +5547,7 @@ NATable::NATable(BindWA *bindWA,
LOB_CLI_SELECT_CURSOR,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,(char *)lobHdfsServer,lobHdfsPort,0);
if (cliRC == 0)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
index 28cd8d7..174cb6f 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLcleanup.cpp
@@ -812,7 +812,8 @@ short CmpSeabaseMDcleanup::dropLOBs(ExeCliInterface *cliInterface)
return 0;
NAString newSchName = "\"" + catName_ + "\"" + "." + "\"" + schName_ + "\"";
-
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
cliRC = SQL_EXEC_LOBddlInterface((char*)newSchName.data(),
newSchName.length(),
objUID_,
@@ -820,7 +821,9 @@ short CmpSeabaseMDcleanup::dropLOBs(ExeCliInterface *cliInterface)
LOB_CLI_CLEANUP,
lobNumList_,
lobTypList_,
- lobLocList_,0);
+ lobLocList_,
+ (char *)lobHdfsServer,
+ lobHdfsPort,0);
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
index d5a0efa..b3e59f8 100644
--- a/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
+++ b/core/sql/sqlcomp/CmpSeabaseDDLtable.cpp
@@ -2190,6 +2190,8 @@ short CmpSeabaseDDL::createSeabaseTable2(
}
Int64 lobMaxSize = CmpCommon::getDefaultNumeric(LOB_MAX_SIZE)*1024*1024;
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
if (j > 0)
{
//if the table is a volatile table return an error
@@ -2221,6 +2223,8 @@ short CmpSeabaseDDL::createSeabaseTable2(
lobNumList,
lobTypList,
lobLocList,
+ (char *)lobHdfsServer,
+ lobHdfsPort,
lobMaxSize);
if (rc < 0)
@@ -3519,6 +3523,8 @@ short CmpSeabaseDDL::dropSeabaseTable2(
short *lobNumList = new (STMTHEAP) short[numCols];
short *lobTypList = new (STMTHEAP) short[numCols];
char **lobLocList = new (STMTHEAP) char*[numCols];
+ const char *lobHdfsServer = CmpCommon::getDefaultString(LOB_HDFS_SERVER);
+ Int32 lobHdfsPort = (Lng32)CmpCommon::getDefaultNumeric(LOB_HDFS_PORT);
Lng32 j = 0;
for (Int32 i = 0; i < nacolArr.entries(); i++)
{
@@ -3539,7 +3545,7 @@ short CmpSeabaseDDL::dropSeabaseTable2(
const char* f = ActiveSchemaDB()->getDefaults().
getValue(LOB_STORAGE_FILE_DIR);
-
+
strcpy(loc, f);
lobLocList[j] = loc;
@@ -3565,7 +3571,7 @@ short CmpSeabaseDDL::dropSeabaseTable2(
LOB_CLI_DROP,
lobNumList,
lobTypList,
- lobLocList,0);
+ lobLocList,(char *)lobHdfsServer, lobHdfsPort,0);
if (rc < 0)
{
*CmpCommon::diags() << DgSqlCode(-CAT_UNABLE_TO_DROP_OBJECT)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/DefaultConstants.h
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/DefaultConstants.h b/core/sql/sqlcomp/DefaultConstants.h
index 1a988b4..c704567 100644
--- a/core/sql/sqlcomp/DefaultConstants.h
+++ b/core/sql/sqlcomp/DefaultConstants.h
@@ -3296,6 +3296,7 @@ enum DefaultConstants
LOB_HDFS_SERVER,
LOB_HDFS_PORT,
+ LOB_GC_LIMIT_SIZE,
// Should the DISK POOL be turned on when replicating the DDL using COPY DDL
REPLICATE_DISK_POOL,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/c29103fd/core/sql/sqlcomp/nadefaults.cpp
----------------------------------------------------------------------
diff --git a/core/sql/sqlcomp/nadefaults.cpp b/core/sql/sqlcomp/nadefaults.cpp
index b4aaa78..2ecbe9e 100644
--- a/core/sql/sqlcomp/nadefaults.cpp
+++ b/core/sql/sqlcomp/nadefaults.cpp
@@ -2068,12 +2068,16 @@ SDDkwd__(ISO_MAPPING, (char *)SQLCHARSETSTRING_ISO88591),
// precision but degraded performance.
SDDkwd__(LIMIT_MAX_NUMERIC_PRECISION, "SYSTEM"),
+ // Size in bytes used to perform garbage collection to lob data file
+ // default size is 5GB . Change to adjust disk usage.
+ DDint__(LOB_GC_LIMIT_SIZE, "5000"),
+
DDint__(LOB_HDFS_PORT, "0"),
DD_____(LOB_HDFS_SERVER, "default"),
-
+
// Size of memoryin bytes used to perform I/O to lob data file
// default size is 512MB . Change to adjust memory usage.
- DDint__(LOB_MAX_CHUNK_MEM_SIZE, "536870912"),
+ DDint__(LOB_MAX_CHUNK_MEM_SIZE, "512"),
// default size is 10 G (10000 M)
DDint__(LOB_MAX_SIZE, "10000"),
// default size is 32000. Change this to extract more data into memory.
@@ -6920,6 +6924,10 @@ DefaultToken NADefaults::token(Int32 attrEnum,
isValid = TRUE;
break;
+ case LOB_GC_LIMIT_SIZE:
+ if (tok >= 0 )
+ isValid=TRUE;
+
case TRAF_TRANS_TYPE:
if (tok == DF_MVCC || tok == DF_SSCC)
isValid = TRUE;