You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by se...@apache.org on 2016/12/19 23:19:00 UTC
[3/8] incubator-trafodion git commit: [TRAFODION-2356] HBase snapshot
concept needs to be contained within HBaseClient in Trafodion
[TRAFODION-2356] HBase snapshot concept needs to be contained within
HBaseClient in Trafodion
Moved snapshot creation/deletion from SequenceFileWriter to HBaseClient.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/1c356227
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/1c356227
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/1c356227
Branch: refs/heads/master
Commit: 1c356227da4706b07962350eea6cfb9c9ad5304c
Parents: e0dab3e
Author: selvaganesang <se...@esgyn.com>
Authored: Tue Nov 22 04:20:29 2016 +0000
Committer: selvaganesang <se...@esgyn.com>
Committed: Tue Nov 22 04:20:29 2016 +0000
----------------------------------------------------------------------
core/sql/executor/ExExeUtil.h | 1 +
core/sql/executor/ExExeUtilLoad.cpp | 45 +++++--
core/sql/executor/ExHbaseAccess.cpp | 30 +++++
core/sql/executor/ExHbaseAccess.h | 1 +
core/sql/executor/HBaseClient_JNI.cpp | 129 +++++++++++++++++--
core/sql/executor/HBaseClient_JNI.h | 12 ++
core/sql/executor/SequenceFileReader.cpp | 127 ------------------
core/sql/executor/SequenceFileReader.h | 12 --
core/sql/exp/ExpHbaseDefs.h | 6 +
core/sql/exp/ExpHbaseInterface.cpp | 45 +++++++
core/sql/exp/ExpHbaseInterface.h | 12 +-
.../java/org/trafodion/sql/HBaseClient.java | 41 ++++++
.../java/org/trafodion/sql/HBulkLoadClient.java | 2 -
.../org/trafodion/sql/SequenceFileWriter.java | 70 +---------
14 files changed, 307 insertions(+), 226 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExExeUtil.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtil.h b/core/sql/executor/ExExeUtil.h
index f3fccb8..73bee38 100755
--- a/core/sql/executor/ExExeUtil.h
+++ b/core/sql/executor/ExExeUtil.h
@@ -3678,6 +3678,7 @@ class ExExeUtilHBaseBulkUnLoadTcb : public ExExeUtilTcb
NAList<struct snapshotStruct *> * snapshotsList_;
NABoolean emptyTarget_;
NABoolean oneFile_;
+ ExpHbaseInterface * ehi_;
};
class ExExeUtilHbaseUnLoadPrivateState : public ex_tcb_private_state
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExExeUtilLoad.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExExeUtilLoad.cpp b/core/sql/executor/ExExeUtilLoad.cpp
index d11f182..73e0d2f 100644
--- a/core/sql/executor/ExExeUtilLoad.cpp
+++ b/core/sql/executor/ExExeUtilLoad.cpp
@@ -66,6 +66,8 @@ using std::ofstream;
#include "ExpLOBexternal.h"
#include "str.h"
#include "ExpHbaseInterface.h"
+#include "ExHbaseAccess.h"
+
///////////////////////////////////////////////////////////////////
ex_tcb * ExExeUtilCreateTableAsTdb::build(ex_globals * glob)
{
@@ -1884,6 +1886,13 @@ ExExeUtilHBaseBulkUnLoadTcb::ExExeUtilHBaseBulkUnLoadTcb(
oneFile_(FALSE)
{
sequenceFileWriter_ = NULL;
+ int jniDebugPort = 0;
+ int jniDebugTimeout = 0;
+ ehi_ = ExpHbaseInterface::newInstance(getGlobals()->getDefaultHeap(),
+ (char*)"", //Later may need to change to hblTdb.server_,
+ (char*)"", //Later may need to change to hblTdb.zkPort_,
+ jniDebugPort,
+ jniDebugTimeout);
qparent_.down->allocatePstate(this);
}
@@ -1911,9 +1920,10 @@ void ExExeUtilHBaseBulkUnLoadTcb::freeResources()
NADELETE(sequenceFileWriter_, SequenceFileWriter, getMyHeap());
sequenceFileWriter_ = NULL;
}
-
-
+ NADELETE(ehi_, ExpHbaseInterface, getGlobals()->getDefaultHeap());
+ ehi_ = NULL;
}
+
ExExeUtilHBaseBulkUnLoadTcb::~ExExeUtilHBaseBulkUnLoadTcb()
{
freeResources();
@@ -2058,7 +2068,7 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
short retcode = 0;
short rc;
SFW_RetCode sfwRetCode = SFW_OK;
-
+ Lng32 hbcRetCode = HBC_OK;
// if no parent request, return
if (qparent_.down->isEmpty())
return WORK_OK;
@@ -2103,6 +2113,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
break;
}
}
+ if ((retcode = ehi_->init(NULL)) != HBASE_ACCESS_SUCCESS)
+ {
+ handleError();
+ step_ = UNLOAD_END_ERROR_;
+ break;
+ }
if (!hblTdb().getOverwriteMergeFile() && hblTdb().getMergePath() != NULL)
{
NABoolean exists = FALSE;
@@ -2234,12 +2250,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
for ( int i = 0 ; i < snapshotsList_->entries(); i++)
{
if (createSnp)
- sfwRetCode = sequenceFileWriter_->createSnapshot( *snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName);
+ hbcRetCode = ehi_->createSnapshot( *snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName);
else
{
NABoolean exist = FALSE;
- sfwRetCode = sequenceFileWriter_->verifySnapshot(*snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName, exist);
- if ( sfwRetCode == SFW_OK && !exist)
+ hbcRetCode = ehi_->verifySnapshot(*snapshotsList_->at(i)->fullTableName, *snapshotsList_->at(i)->snapshotName, exist);
+ if ( hbcRetCode == HBC_OK && !exist)
{
ComDiagsArea * da = getDiagsArea();
*da << DgSqlCode(-8112)
@@ -2249,9 +2265,12 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
break;
}
}
- if (sfwRetCode != SFW_OK)
+ if (hbcRetCode != HBC_OK)
{
- createHdfsFileError(sfwRetCode);
+ ExHbaseAccessTcb::setupError((NAHeap *)getMyHeap(),qparent_, hbcRetCode,
+ "HBaseClient_JNI::createSnapshot/verifySnapshot",
+ snapshotsList_->at(i)->snapshotName->data() );
+ handleError();
step_ = UNLOAD_END_ERROR_;
break;
}
@@ -2307,10 +2326,13 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
return rc;
for ( int i = 0 ; i < snapshotsList_->entries(); i++)
{
- sfwRetCode = sequenceFileWriter_->deleteSnapshot( *snapshotsList_->at(i)->snapshotName);
- if (sfwRetCode != SFW_OK)
+ hbcRetCode = ehi_->deleteSnapshot( *snapshotsList_->at(i)->snapshotName);
+ if (hbcRetCode != HBC_OK)
{
- createHdfsFileError(sfwRetCode);
+ ExHbaseAccessTcb::setupError((NAHeap *)getMyHeap(),qparent_, hbcRetCode,
+ "HBaseClient_JNI::createSnapshot/verifySnapshot",
+ snapshotsList_->at(i)->snapshotName->data() );
+ handleError();
step_ = UNLOAD_END_ERROR_;
break;
}
@@ -2351,6 +2373,7 @@ short ExExeUtilHBaseBulkUnLoadTcb::work()
case UNLOAD_END_:
case UNLOAD_END_ERROR_:
{
+ ehi_->close();
if (restoreCQD("TRAF_TABLE_SNAPSHOT_SCAN") < 0)
{
step_ = UNLOAD_ERROR_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.cpp b/core/sql/executor/ExHbaseAccess.cpp
index 53a4dd1..a7b7f4e 100644
--- a/core/sql/executor/ExHbaseAccess.cpp
+++ b/core/sql/executor/ExHbaseAccess.cpp
@@ -686,6 +686,36 @@ short ExHbaseAccessTcb::moveRowToUpQueue(const char * row, Lng32 len,
return 0;
}
+short ExHbaseAccessTcb::setupError(NAHeap *heap, ex_queue_pair &qparent, Lng32 retcode, const char * str, const char * str2)
+{
+ ContextCli *currContext = GetCliGlobals()->currContext();
+ // Make sure retcode is positive.
+ if (retcode < 0)
+ retcode = -retcode;
+
+ if ((ABS(retcode) >= HBASE_MIN_ERROR_NUM) &&
+ (ABS(retcode) <= HBASE_MAX_ERROR_NUM))
+ {
+ ex_queue_entry *pentry_down = qparent.down->getHeadEntry();
+
+ Lng32 cliError = 0;
+
+ Lng32 intParam1 = -retcode;
+ ComDiagsArea * diagsArea = NULL;
+ ExRaiseSqlError(heap, &diagsArea,
+ (ExeErrorCode)(8448), NULL, &intParam1,
+ &cliError, NULL,
+ (str ? (char*)str : (char*)" "),
+ getHbaseErrStr(retcode),
+ (str2 ? (char*)str2 :
+ (char *)currContext->getJniErrorStr().data()));
+ pentry_down->setDiagsArea(diagsArea);
+ return -1;
+ }
+
+ return 0;
+}
+
short ExHbaseAccessTcb::setupError(Lng32 retcode, const char * str, const char * str2)
{
ContextCli *currContext = GetCliGlobals()->currContext();
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/ExHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseAccess.h b/core/sql/executor/ExHbaseAccess.h
index eb0792c..03cb052 100644
--- a/core/sql/executor/ExHbaseAccess.h
+++ b/core/sql/executor/ExHbaseAccess.h
@@ -187,6 +187,7 @@ public:
const char * loggingFileNamePrefix,
Lng32 instId,
char * loggingFileName);
+ static short setupError(NAHeap *heap, ex_queue_pair &qparent, Lng32 retcode, const char * str, const char * str2 = NULL);
protected:
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/HBaseClient_JNI.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp
index 272ee5b..603c42e 100644
--- a/core/sql/executor/HBaseClient_JNI.cpp
+++ b/core/sql/executor/HBaseClient_JNI.cpp
@@ -99,6 +99,12 @@ static const char* const hbcErrorEnumStr[] =
,"Preparing parameters for getKeys()."
,"Preparing parameters for listAll()."
,"Preparing parameters for getRegionStats()."
+ ,"JNI NewStringUTF() in createSnapshot()."
+ ,"Java exception in createSnapshot()."
+ ,"JNI NewStringUTF() in deleteSnapshot()."
+ ,"Java exception in deleteSnapshot()."
+ ,"JNI NewStringUTF() in verifySnapshot()."
+ ,"Java exception in verifySnapshot()."
};
//////////////////////////////////////////////////////////////////////////////
@@ -281,6 +287,12 @@ HBC_RetCode HBaseClient_JNI::init()
JavaMethods_[JM_HBC_GETSTARTKEYS ].jm_signature = "(Ljava/lang/String;Z)[[B";
JavaMethods_[JM_HBC_GETENDKEYS ].jm_name = "getEndKeys";
JavaMethods_[JM_HBC_GETENDKEYS ].jm_signature = "(Ljava/lang/String;Z)[[B";
+ JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_name = "createSnapshot";
+ JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
+ JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_name = "deleteSnapshot";
+ JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_signature = "(Ljava/lang/String;)Z";
+ JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_name = "verifySnapshot";
+ JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
rc = (HBC_RetCode)JavaObjectInterface::init(className, javaClass_, JavaMethods_, (Int32)JM_LAST, javaMethodsInitialized_);
javaMethodsInitialized_ = TRUE;
pthread_mutex_unlock(&javaMethodsInitMutex_);
@@ -1814,10 +1826,8 @@ HVC_RetCode HiveClient_JNI::hdfsCreateFile(const char* path)
{
QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HiveClient_JNI::hdfsCreate(%s) called.", path);
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return HVC_ERROR_HDFS_CREATE_EXCEPTION;
- }
+ if (initJNIEnv() != JOI_OK)
+ return HVC_ERROR_INIT_PARAM;
jstring js_path = jenv_->NewStringUTF(path);
if (js_path == NULL) {
@@ -1852,10 +1862,8 @@ HVC_RetCode HiveClient_JNI::hdfsCreateFile(const char* path)
{
QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HiveClient_JNI::hdfsWrite(%ld) called.", len);
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return HVC_ERROR_HDFS_WRITE_EXCEPTION;
- }
+ if (initJNIEnv() != JOI_OK)
+ return HVC_ERROR_INIT_PARAM;
//Write the requisite bytes into the file
jbyteArray jbArray = jenv_->NewByteArray( len);
@@ -3027,7 +3035,112 @@ NAArray<HbaseStr>* HBaseClient_JNI::getKeys(Int32 funcIndex, NAHeap *heap, const
return retArray;
}
+HBC_RetCode HBaseClient_JNI::createSnapshot( const NAString& tableName, const NAString& snapshotName)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::createSnapshot(%s, %s) called.",
+ tableName.data(), snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_tableName = jenv_->NewStringUTF(tableName.data());
+ if (js_tableName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_CREATE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_PARAM;
+ }
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_CREATE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_CREATE_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_CREATE_SNAPSHOT].methodID, js_tableName, js_snapshotName);
+
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::createSnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_CREATE_SNAPSHOT_EXCEPTION;
+ }
+
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
+
+HBC_RetCode HBaseClient_JNI::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::verifySnapshot(%s, %s) called.",
+ tableName.data(), snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_tableName = jenv_->NewStringUTF(tableName.data());
+ if (js_tableName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_VERIFY_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_PARAM;
+ }
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_VERIFY_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_VERIFY_SNAPSHOT].methodID, js_tableName, js_snapshotName);
+
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::verifySnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
+ }
+
+ exist = jresult;
+
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
+
+HBC_RetCode HBaseClient_JNI::deleteSnapshot( const NAString& snapshotName)
+{
+ QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "HBaseClient_JNI::deleteSnapshot(%s) called.",
+ snapshotName.data());
+
+ if (initJNIEnv() != JOI_OK)
+ return HBC_ERROR_INIT_PARAM;
+
+ jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
+ if (js_snapshotName == NULL) {
+ GetCliGlobals()->setJniErrorStr(getErrorText(HBC_ERROR_DELETE_SNAPSHOT_PARAM));
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_DELETE_SNAPSHOT_PARAM;
+ }
+
+ tsRecentJMFromJNI = JavaMethods_[JM_HBC_DELETE_SNAPSHOT].jm_full_name;
+ jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DELETE_SNAPSHOT].methodID, js_snapshotName);
+ if (jenv_->ExceptionCheck())
+ {
+ getExceptionDetails();
+ logError(CAT_SQL_HBASE, __FILE__, __LINE__);
+ logError(CAT_SQL_HBASE, "HBaseClient_JNI::deleteSnapshot()", getLastError());
+ jenv_->PopLocalFrame(NULL);
+ return HBC_ERROR_DELETE_SNAPSHOT_EXCEPTION;
+ }
+ jenv_->PopLocalFrame(NULL);
+ return HBC_OK;
+}
// ===========================================================================
// ===== Class HTableClient
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/HBaseClient_JNI.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.h b/core/sql/executor/HBaseClient_JNI.h
index d0eeeb2..08464ab 100644
--- a/core/sql/executor/HBaseClient_JNI.h
+++ b/core/sql/executor/HBaseClient_JNI.h
@@ -398,6 +398,12 @@ typedef enum {
,HBC_ERROR_GETKEYS
,HBC_ERROR_LISTALL
,HBC_ERROR_REGION_STATS
+ ,HBC_ERROR_CREATE_SNAPSHOT_PARAM
+ ,HBC_ERROR_CREATE_SNAPSHOT_EXCEPTION
+ ,HBC_ERROR_DELETE_SNAPSHOT_PARAM
+ ,HBC_ERROR_DELETE_SNAPSHOT_EXCEPTION
+ ,HBC_ERROR_VERIFY_SNAPSHOT_PARAM
+ ,HBC_ERROR_VERIFY_SNAPSHOT_EXCEPTION
,HBC_LAST
} HBC_RetCode;
@@ -508,6 +514,9 @@ public:
HTableClient_JNI **outHtc);
NAArray<HbaseStr>* getStartKeys(NAHeap *heap, const char *tableName, bool useTRex);
NAArray<HbaseStr>* getEndKeys(NAHeap *heap, const char * tableName, bool useTRex);
+ HBC_RetCode createSnapshot( const NAString& tableName, const NAString& snapshotName);
+ HBC_RetCode deleteSnapshot( const NAString& snapshotName);
+ HBC_RetCode verifySnapshot( const NAString& tableName, const NAString& snapshotName, NABoolean & exist);
private:
// private default constructor
@@ -558,6 +567,9 @@ private:
,JM_HBC_CHECKANDDELETE_ROW
,JM_HBC_GETSTARTKEYS
,JM_HBC_GETENDKEYS
+ ,JM_HBC_CREATE_SNAPSHOT
+ ,JM_HBC_DELETE_SNAPSHOT
+ ,JM_HBC_VERIFY_SNAPSHOT
,JM_LAST
};
static jclass javaClass_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/SequenceFileReader.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/SequenceFileReader.cpp b/core/sql/executor/SequenceFileReader.cpp
index af31931..aff2945 100644
--- a/core/sql/executor/SequenceFileReader.cpp
+++ b/core/sql/executor/SequenceFileReader.cpp
@@ -455,13 +455,7 @@ static const char* const sfwErrorEnumStr[] =
,"JNI NewStringUTF() in hdfsExists()."
,"Java exception in hdfsExists()."
,"file already exists."
- ,"JNI NewStringUTF() in createSnapshot()."
- ,"Java exception in createSnapshot()."
- ,"JNI NewStringUTF() in deleteSnapshot()."
- ,"Java exception in deleteSnapshot()."
,"Java exception in release()."
- ,"JNI NewStringUTF() in verifySnapshot()."
- ,"Java exception in verifySnapshot()."
,"JNI NewStringUTF() in hdfsDeletePath()."
,"Java exception in hdfsDeletePath()."
};
@@ -526,12 +520,6 @@ SFW_RetCode SequenceFileWriter::init()
JavaMethods_[JM_HDFS_CLEAN_UNLOAD_PATH].jm_signature = "(Ljava/lang/String;)Z";
JavaMethods_[JM_HDFS_EXISTS].jm_name = "hdfsExists";
JavaMethods_[JM_HDFS_EXISTS].jm_signature = "(Ljava/lang/String;)Z";
- JavaMethods_[JM_CREATE_SNAPSHOT].jm_name = "createSnapshot";
- JavaMethods_[JM_CREATE_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
- JavaMethods_[JM_DELETE_SNAPSHOT].jm_name = "deleteSnapshot";
- JavaMethods_[JM_DELETE_SNAPSHOT].jm_signature = "(Ljava/lang/String;)Z";
- JavaMethods_[JM_VERIFY_SNAPSHOT].jm_name = "verifySnapshot";
- JavaMethods_[JM_VERIFY_SNAPSHOT].jm_signature = "(Ljava/lang/String;Ljava/lang/String;)Z";
JavaMethods_[JM_HDFS_DELETE_PATH].jm_name = "hdfsDeletePath";
JavaMethods_[JM_HDFS_DELETE_PATH].jm_signature = "(Ljava/lang/String;)Z";
@@ -808,121 +796,6 @@ SFW_RetCode SequenceFileWriter::hdfsCleanUnloadPath( const NAString& uldPath)
return SFW_OK;
}
-SFW_RetCode SequenceFileWriter::createSnapshot( const NAString& tableName, const NAString& snapshotName)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::createSnapshot(%s, %s) called.",
- tableName.data(), snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_tableName = jenv_->NewStringUTF(tableName.data());
- if (js_tableName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_CREATE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_PARAM;
- }
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_CREATE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_CREATE_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_CREATE_SNAPSHOT].methodID, js_tableName, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::createSnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION;
- }
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-SFW_RetCode SequenceFileWriter::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
- NABoolean & exist)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::verifySnapshot(%s, %s) called.",
- tableName.data(), snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_tableName = jenv_->NewStringUTF(tableName.data());
- if (js_tableName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_VERIFY_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_PARAM;
- }
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_VERIFY_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_VERIFY_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_VERIFY_SNAPSHOT].methodID, js_tableName, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::verifySnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION;
- }
-
- exist = jresult;
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-SFW_RetCode SequenceFileWriter::deleteSnapshot( const NAString& snapshotName)
-{
- QRLogger::log(CAT_SQL_HBASE, LL_DEBUG, "SequenceFileWriter::deleteSnapshot(%s) called.",
- snapshotName.data());
-
- if (jenv_->PushLocalFrame(jniHandleCapacity_) != 0) {
- getExceptionDetails();
- return SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION;
- }
-
- jstring js_snapshotName= jenv_->NewStringUTF(snapshotName.data());
- if (js_snapshotName == NULL) {
- GetCliGlobals()->setJniErrorStr(getErrorText(SFW_ERROR_DELETE_SNAPSHOT_PARAM));
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_DELETE_SNAPSHOT_PARAM;
- }
-
- tsRecentJMFromJNI = JavaMethods_[JM_DELETE_SNAPSHOT].jm_full_name;
- jboolean jresult = jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_DELETE_SNAPSHOT].methodID, js_snapshotName);
-
- if (jenv_->ExceptionCheck())
- {
- getExceptionDetails();
- logError(CAT_SQL_HBASE, __FILE__, __LINE__);
- logError(CAT_SQL_HBASE, "SequenceFileWriter::deleteSnapshot()", getLastError());
- jenv_->PopLocalFrame(NULL);
- return SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION;
- }
-
- jenv_->PopLocalFrame(NULL);
- return SFW_OK;
-}
-
-
SFW_RetCode SequenceFileWriter::hdfsMergeFiles( const NAString& srcPath,
const NAString& dstPath)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/executor/SequenceFileReader.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/SequenceFileReader.h b/core/sql/executor/SequenceFileReader.h
index 9180de8..d950677 100644
--- a/core/sql/executor/SequenceFileReader.h
+++ b/core/sql/executor/SequenceFileReader.h
@@ -156,12 +156,6 @@ typedef enum {
,SFW_ERROR_HDFS_EXISTS_PARAM
,SFW_ERROR_HDFS_EXISTS_EXCEPTION
,SFW_ERROR_HDFS_EXISTS_FILE_EXISTS
- ,SFW_ERROR_CREATE_SNAPSHOT_PARAM
- ,SFW_ERROR_CREATE_SNAPSHOT_EXCEPTION
- ,SFW_ERROR_DELETE_SNAPSHOT_PARAM
- ,SFW_ERROR_DELETE_SNAPSHOT_EXCEPTION
- ,SFW_ERROR_VERIFY_SNAPSHOT_PARAM
- ,SFW_ERROR_VERIFY_SNAPSHOT_EXCEPTION
,SFW_ERROR_HDFS_DELETE_PATH_PARAM
,SFW_ERROR_HDFS_DELETE_PATH_EXCEPTION
,SFW_LAST
@@ -214,9 +208,6 @@ public:
SFW_RetCode hdfsCleanUnloadPath(const NAString& uldPath );
SFW_RetCode hdfsExists(const NAString& uldPath, NABoolean & exists );
SFW_RetCode hdfsClose();
- SFW_RetCode createSnapshot( const NAString& tableName, const NAString& snapshotName);
- SFW_RetCode deleteSnapshot( const NAString& snapshotName);
- SFW_RetCode verifySnapshot( const NAString& tableName, const NAString& snapshotName, NABoolean & exist);
SFW_RetCode release();
virtual char* getErrorText(SFW_RetCode errEnum);
@@ -237,9 +228,6 @@ private:
JM_HDFS_MERGE_FILES,
JM_HDFS_CLEAN_UNLOAD_PATH,
JM_HDFS_EXISTS,
- JM_CREATE_SNAPSHOT,
- JM_DELETE_SNAPSHOT,
- JM_VERIFY_SNAPSHOT,
JM_HDFS_DELETE_PATH,
JM_LAST
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseDefs.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseDefs.h b/core/sql/exp/ExpHbaseDefs.h
index 896735c..16690bf 100644
--- a/core/sql/exp/ExpHbaseDefs.h
+++ b/core/sql/exp/ExpHbaseDefs.h
@@ -130,6 +130,9 @@ typedef enum
HBASE_CLEANUP_HFILE_ERROR,
HBASE_INIT_HBLC_ERROR,
HBASE_RETRY_AGAIN,
+ HBASE_CREATE_SNAPSHOT_ERROR,
+ HBASE_DELETE_SNAPSHOT_ERROR,
+ HBASE_VERIFY_SNAPSHOT_ERROR,
HBASE_MAX_ERROR_NUM // keep this as the last element in enum list.
} HbaseError;
@@ -154,6 +157,9 @@ static const char * const hbaseErrorEnumStr[] =
"HBASE_DOBULK_LOAD_ERROR",
"HBASE_CLEANUP_HFILE_ERROR",
"HBASE_INIT_HBLC_ERROR",
+ "HBASE_CREATE_SNAPSHOT_ERROR",
+ "HBASE_DELETE_SNAPSHOT_ERROR",
+ "HBASE_VERIFY_SNAPSHOT_ERROR",
"HBASE_MAX_ERROR_NUM" // keep this as the last element in enum list.
};
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.cpp b/core/sql/exp/ExpHbaseInterface.cpp
index 0b5a77f..994d8fa 100644
--- a/core/sql/exp/ExpHbaseInterface.cpp
+++ b/core/sql/exp/ExpHbaseInterface.cpp
@@ -1588,3 +1588,48 @@ NAArray<HbaseStr> * ExpHbaseInterface_JNI::getClusterStats(Int32 &numEntries)
return regionStats;
}
+Lng32 ExpHbaseInterface_JNI::createSnapshot( const NAString& tableName, const NAString& snapshotName)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->createSnapshot(tableName, snapshotName);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_CREATE_SNAPSHOT_ERROR;
+}
+
+Lng32 ExpHbaseInterface_JNI::deleteSnapshot( const NAString& snapshotName)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->deleteSnapshot(snapshotName);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_DELETE_SNAPSHOT_ERROR;
+}
+
+Lng32 ExpHbaseInterface_JNI::verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist)
+{
+ if (client_ == NULL)
+ {
+ if (init(hbs_) != HBASE_ACCESS_SUCCESS)
+ return -HBASE_ACCESS_ERROR;
+ }
+
+ retCode_ = client_->verifySnapshot(tableName, snapshotName, exist);
+ if (retCode_ == HBC_OK)
+ return HBASE_ACCESS_SUCCESS;
+ else
+ return HBASE_VERIFY_SNAPSHOT_ERROR;
+}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/exp/ExpHbaseInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.h b/core/sql/exp/ExpHbaseInterface.h
index bf4a355..1d7728c 100644
--- a/core/sql/exp/ExpHbaseInterface.h
+++ b/core/sql/exp/ExpHbaseInterface.h
@@ -367,12 +367,17 @@ class ExpHbaseInterface : public NABasicObject
virtual Lng32 getRegionsNodeName(const HbaseStr& tblName,
Int32 partns,
ARRAY(const char *)& nodeNames) = 0;
-
// get regions and size
virtual NAArray<HbaseStr> *getRegionStats(const HbaseStr& tblName) = 0;
virtual NAArray<HbaseStr> *getClusterStats(Int32 &numEntries) = 0;
+ // Snapshots
+ virtual Lng32 createSnapshot( const NAString& tableName, const NAString& snapshotName) = 0;
+ virtual Lng32 deleteSnapshot( const NAString& snapshotName) = 0;
+ virtual Lng32 verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist) = 0;
+
protected:
enum
{
@@ -678,6 +683,11 @@ virtual Lng32 initHFileParams(HbaseStr &tblName,
virtual NAArray<HbaseStr>* getRegionStats(const HbaseStr& tblName);
virtual NAArray<HbaseStr>* getClusterStats(Int32 &numEntries);
+ virtual Lng32 createSnapshot( const NAString& tableName, const NAString& snapshotName);
+ virtual Lng32 deleteSnapshot( const NAString& snapshotName);
+ virtual Lng32 verifySnapshot( const NAString& tableName, const NAString& snapshotName,
+ NABoolean & exist);
+
private:
bool useTRex_;
HBaseClient_JNI* client_;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
index c007f74..7bf02b2 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
@@ -1757,6 +1757,47 @@ public class HBaseClient {
return endKeys;
}
+ public boolean createSnapshot( String tableName, String snapshotName)
+ throws IOException
+ {
+ Admin admin = connection.getAdmin();
+ admin.snapshot(snapshotName, TableName.valueOf(tableName));
+ admin.close();
+ if (logger.isDebugEnabled()) logger.debug("HBaseClient.createSnapshot() - Snapshot created: " + snapshotName);
+ return true;
+ }
+
+ public boolean verifySnapshot( String tableName, String snapshotName)
+ throws IOException
+ {
+ Admin admin = connection.getAdmin();
+ List<SnapshotDescription> lstSnaps = admin.listSnapshots();
+ try
+ {
+ for (SnapshotDescription snpd : lstSnaps) {
+ if (snpd.getName().compareTo(snapshotName) == 0 &&
+ snpd.getTable().compareTo(tableName) == 0) {
+ if (logger.isDebugEnabled())
+ logger.debug("HBaseClient.verifySnapshot() - Snapshot verified: " + snapshotName);
+ return true;
+ }
+ }
+ } finally {
+ admin.close();
+ }
+ return false;
+ }
+
+ public boolean deleteSnapshot( String snapshotName)
+ throws MasterNotRunningException, IOException, SnapshotCreationException,
+ InterruptedException, ZooKeeperConnectionException, ServiceException
+ {
+ Admin admin = connection.getAdmin();
+ admin.deleteSnapshot(snapshotName);
+ admin.close();
+ if (logger.isDebugEnabled()) logger.debug("HBaseClient.deleteSnapshot() - Snapshot deleted: " + snapshotName);
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java b/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
index 4bd104f..9d51d37 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBulkLoadClient.java
@@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.trafodion.sql.HTableClient;
-//import org.trafodion.sql.HBaseClient;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.io.compress.CodecPool;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/1c356227/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java b/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
index 28869fe..e1ee33c 100644
--- a/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
+++ b/core/sql/src/main/java/org/trafodion/sql/SequenceFileWriter.java
@@ -34,13 +34,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.MasterNotRunningException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.TableSnapshotScanner;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
@@ -76,8 +70,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
public class SequenceFileWriter {
static Logger logger = Logger.getLogger(SequenceFileWriter.class.getName());
- Configuration conf = null; // File system configuration
- private Connection connection;
+ static Configuration conf = null; // File system configuration
SequenceFile.Writer writer = null;
@@ -89,13 +82,12 @@ public class SequenceFileWriter {
/**
* Class Constructor
*/
- SequenceFileWriter() throws MasterNotRunningException, ZooKeeperConnectionException, ServiceException, IOException
+ static {
+ conf = new Configuration(true);
+ }
+ SequenceFileWriter() throws IOException
{
- init("", "");
- conf = connection.getConfiguration();
- conf.set("fs.hdfs.impl","org.apache.hadoop.hdfs.DistributedFileSystem");
}
-
public String open(String path) throws IOException {
Path filename = new Path(path);
@@ -304,56 +296,4 @@ public class SequenceFileWriter {
fs.delete(delPath, true);
return true;
}
-
- private boolean init(String zkServers, String zkPort) throws IOException
- , ServiceException
- {
- logger.debug("SequenceFileWriter.init(" + zkServers + ", " + zkPort + ") called.");
- connection = HBaseClient.getConnection();
- return true;
- }
-
- public boolean createSnapshot( String tableName, String snapshotName)
- throws IOException
- {
- Admin admin = connection.getAdmin();
- admin.snapshot(snapshotName, TableName.valueOf(tableName));
- admin.close();
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.createSnapshot() - Snapshot created: " + snapshotName);
- return true;
- }
-
- public boolean verifySnapshot( String tableName, String snapshotName)
- throws IOException
- {
- Admin admin = connection.getAdmin();
- List<SnapshotDescription> lstSnaps = admin.listSnapshots();
- try
- {
- for (SnapshotDescription snpd : lstSnaps)
- {
- if (snpd.getName().compareTo(snapshotName) == 0 &&
- snpd.getTable().compareTo(tableName) == 0)
- {
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.verifySnapshot() - Snapshot verified: " + snapshotName);
- return true;
- }
- }
- } finally {
- admin.close();
- }
- return false;
- }
-
- public boolean deleteSnapshot( String snapshotName)
- throws MasterNotRunningException, IOException, SnapshotCreationException,
- InterruptedException, ZooKeeperConnectionException, ServiceException
- {
- Admin admin = connection.getAdmin();
- admin.deleteSnapshot(snapshotName);
- admin.close();
- if (logger.isDebugEnabled()) logger.debug("SequenceFileWriter.deleteSnapshot() - Snapshot deleted: " + snapshotName);
- return true;
- }
-
}