You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@trafodion.apache.org by su...@apache.org on 2017/10/18 19:14:53 UTC
[1/2] incubator-trafodion git commit: [TRAFODION-2775] Insert does
not raise duplicate row error for hbase format table with defaulted first
column
Repository: incubator-trafodion
Updated Branches:
refs/heads/master d283c95ee -> c66fca250
[TRAFODION-2775] Insert does not raise duplicate row error for hbase format table with defaulted first column
There was a problem with column being checked during CheckAndPut call for such tables. We were always checking the first column in the row, which may not have existed in the previous version of this rowid. Please see JIRA for a complete explanation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/921c0a58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/921c0a58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/921c0a58
Branch: refs/heads/master
Commit: 921c0a5840a7f6f233166b1d745a4a7f65f49118
Parents: e8bdaba
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Oct 18 14:51:34 2017 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Oct 18 14:51:34 2017 +0000
----------------------------------------------------------------------
core/sql/comexe/ComTdbHbaseAccess.cpp | 2 ++
core/sql/comexe/ComTdbHbaseAccess.h | 12 ++++++++-
core/sql/executor/ExHbaseIUD.cpp | 6 +++--
core/sql/executor/HBaseClient_JNI.cpp | 7 +++---
core/sql/executor/HBaseClient_JNI.h | 1 +
core/sql/exp/ExpHbaseInterface.cpp | 9 ++++---
core/sql/exp/ExpHbaseInterface.h | 6 +++--
core/sql/generator/GenRelUpdate.cpp | 10 ++++++++
.../java/org/trafodion/sql/HBaseClient.java | 7 ++++--
.../java/org/trafodion/sql/HTableClient.java | 26 +++++++++++---------
10 files changed, 61 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/comexe/ComTdbHbaseAccess.cpp
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHbaseAccess.cpp b/core/sql/comexe/ComTdbHbaseAccess.cpp
index 272fba6..07247fa 100644
--- a/core/sql/comexe/ComTdbHbaseAccess.cpp
+++ b/core/sql/comexe/ComTdbHbaseAccess.cpp
@@ -198,6 +198,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess(
sampleLocation_(NULL),
hbaseRowsetVsbbSize_(0),
trafLoadFlushSize_(0),
+ colIndexOfPK1_(0),
hbaseAccessOptions_(hbaseAccessOptions),
pkeyColName_(pkeyColName)
@@ -311,6 +312,7 @@ ComTdbHbaseAccess::ComTdbHbaseAccess(
sampleLocation_(NULL),
hbaseRowsetVsbbSize_(0),
trafLoadFlushSize_(0),
+ colIndexOfPK1_(0),
hbaseAccessOptions_(NULL),
pkeyColName_(NULL)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/comexe/ComTdbHbaseAccess.h
----------------------------------------------------------------------
diff --git a/core/sql/comexe/ComTdbHbaseAccess.h b/core/sql/comexe/ComTdbHbaseAccess.h
index b84c0c4..64c5217 100644
--- a/core/sql/comexe/ComTdbHbaseAccess.h
+++ b/core/sql/comexe/ComTdbHbaseAccess.h
@@ -882,6 +882,12 @@ public:
UInt16 getTrafLoadFlushSize()
{ return trafLoadFlushSize_; }
+ void setColIndexOfPK1(Int16 val)
+ { colIndexOfPK1_ = val; }
+ Int16 getColIndexOfPK1()
+ { return colIndexOfPK1_; }
+
+
void setLogErrorRows(NABoolean v)
{(v ? flags2_ |= TRAF_LOAD_LOG_ERROR_ROWS : flags2_ &= ~TRAF_LOAD_LOG_ERROR_ROWS); };
NABoolean getLogErrorRows() { return (flags2_ & TRAF_LOAD_LOG_ERROR_ROWS) != 0; };
@@ -1050,8 +1056,12 @@ public:
// col name of primary key. Format colFam:colName.
// Currently used for HbaseMapped tables with a single primary key.
NABasicPtr pkeyColName_;
+ // 0-based column index of first primary key column for hbase format tables
+ // has value 0 for aligned format and native hbase access. This is the index
+ // number of column in row being inserted, not of the column in table.
+ Int16 colIndexOfPK1_ ;
- char fillers[2];
+ char fillers[4];
};
class ComTdbHbaseCoProcAccess : public ComTdbHbaseAccess
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/executor/ExHbaseIUD.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/ExHbaseIUD.cpp b/core/sql/executor/ExHbaseIUD.cpp
index 9fc15bd..bd6307b 100644
--- a/core/sql/executor/ExHbaseIUD.cpp
+++ b/core/sql/executor/ExHbaseIUD.cpp
@@ -541,7 +541,8 @@ ExWorkProcRetcode ExHbaseAccessInsertSQTcb::work()
hbaseAccessTdb().useHbaseXn(),
hbaseAccessTdb().useRegionXn(),
insColTSval_,
- asyncOperation_);
+ asyncOperation_,
+ hbaseAccessTdb().getColIndexOfPK1());
if (retcode == HBASE_DUP_ROW_ERROR) // row exists, return error
{
@@ -2272,7 +2273,8 @@ ExWorkProcRetcode ExHbaseUMDtrafUniqueTaskTcb::work(short &rc)
tcb_->hbaseAccessTdb().useHbaseXn(),
tcb_->hbaseAccessTdb().useRegionXn(),
-1, // colTS
- tcb_->asyncOperation_);
+ tcb_->asyncOperation_,
+ tcb_->hbaseAccessTdb().getColIndexOfPK1());
if (retcode == HBASE_DUP_ROW_ERROR)
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/executor/HBaseClient_JNI.cpp
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp
index 6700cff..78c4289 100644
--- a/core/sql/executor/HBaseClient_JNI.cpp
+++ b/core/sql/executor/HBaseClient_JNI.cpp
@@ -277,7 +277,7 @@ HBC_RetCode HBaseClient_JNI::init()
JavaMethods_[JM_GET_REGN_NODES].jm_name = "getRegionsNodeName";
JavaMethods_[JM_GET_REGN_NODES].jm_signature = "(Ljava/lang/String;[Ljava/lang/String;)Z";
JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].jm_name = "insertRow";
- JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].jm_signature = "(JLjava/lang/String;ZJ[BLjava/lang/Object;JZZZ)Z";
+ JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].jm_signature = "(JLjava/lang/String;ZJ[BLjava/lang/Object;JZSZZ)Z";
JavaMethods_[JM_HBC_DIRECT_INSERT_ROWS].jm_name = "insertRows";
JavaMethods_[JM_HBC_DIRECT_INSERT_ROWS].jm_signature = "(JLjava/lang/String;ZJSLjava/lang/Object;Ljava/lang/Object;JZ)Z";
JavaMethods_[JM_HBC_DIRECT_CHECKANDUPDATE_ROW].jm_name = "checkAndUpdateRow";
@@ -2460,7 +2460,7 @@ HBC_RetCode HBaseClient_JNI::getHbaseTableInfo(const char* tblName,
HBC_RetCode HBaseClient_JNI::insertRow(NAHeap *heap, const char *tableName,
ExHbaseAccessStats *hbs, bool useTRex, Int64 transID, HbaseStr rowID,
HbaseStr row, Int64 timestamp, bool checkAndPut, bool asyncOperation,
- bool useRegionXn, HTableClient_JNI **outHtc)
+ bool useRegionXn, short colIndexToCheck, HTableClient_JNI **outHtc)
{
HTableClient_JNI *htc = NULL;
@@ -2513,13 +2513,14 @@ HBC_RetCode HBaseClient_JNI::insertRow(NAHeap *heap, const char *tableName,
jboolean j_checkAndPut = checkAndPut;
jboolean j_asyncOperation = asyncOperation;
jboolean j_useRegionXn = useRegionXn;
+ jshort j_colIndexToCheck = colIndexToCheck;
if (hbs)
hbs->getHbaseTimer().start();
tsRecentJMFromJNI = JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].jm_full_name;
jboolean jresult =
jenv_->CallBooleanMethod(javaObj_, JavaMethods_[JM_HBC_DIRECT_INSERT_ROW].methodID,
- j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, jRow, j_ts, j_checkAndPut, j_asyncOperation, j_useRegionXn);
+ j_htc, js_tblName, j_useTRex, j_tid, jba_rowID, jRow, j_ts, j_checkAndPut, j_colIndexToCheck, j_asyncOperation, j_useRegionXn);
if (hbs) {
hbs->incHbaseCalls();
if (!asyncOperation)
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/executor/HBaseClient_JNI.h
----------------------------------------------------------------------
diff --git a/core/sql/executor/HBaseClient_JNI.h b/core/sql/executor/HBaseClient_JNI.h
index 09e495a..94e8922 100644
--- a/core/sql/executor/HBaseClient_JNI.h
+++ b/core/sql/executor/HBaseClient_JNI.h
@@ -496,6 +496,7 @@ public:
HbaseStr rowID,
HbaseStr row, Int64 timestamp,bool checkAndPut,
bool asyncOperation, bool useRegionXn,
+ short colIndexToCheck,
HTableClient_JNI **outHtc);
HBC_RetCode insertRows(NAHeap *heap, const char *tableName,
ExHbaseAccessStats *hbs, bool useTRex, Int64 transID, short rowIDLen, HbaseStr rowIDs,
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/exp/ExpHbaseInterface.cpp
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.cpp b/core/sql/exp/ExpHbaseInterface.cpp
index 44ce725..dff16dc 100644
--- a/core/sql/exp/ExpHbaseInterface.cpp
+++ b/core/sql/exp/ExpHbaseInterface.cpp
@@ -842,7 +842,9 @@ Lng32 ExpHbaseInterface_JNI::insertRow(
transID = getTransactionIDFromContext();
retCode_ = client_->insertRow((NAHeap *)heap_, tblName.val, hbs_,
useTRex_, transID, rowID, row, timestamp,
- checkAndPut, asyncOperation, useRegionXn, &htc);
+ checkAndPut, asyncOperation, useRegionXn,
+ 0, // checkAndPut is false, so colIndexToCheck is not used
+ &htc);
if (retCode_ != HBC_OK) {
return -HBASE_ACCESS_ERROR;
}
@@ -1194,7 +1196,8 @@ Lng32 ExpHbaseInterface_JNI::checkAndInsertRow(
NABoolean useHbaseXn,
NABoolean useRegionXn,
const int64_t timestamp,
- NABoolean asyncOperation)
+ NABoolean asyncOperation,
+ Int16 colIndexToCheck)
{
HTableClient_JNI *htc = NULL;
Int64 transID;
@@ -1207,7 +1210,7 @@ Lng32 ExpHbaseInterface_JNI::checkAndInsertRow(
retCode_ = client_->insertRow((NAHeap *)heap_, tblName.val, hbs_,
useTRex_, transID, rowID, row, timestamp,
checkAndPut, asyncOperation, useRegionXn,
- &htc);
+ colIndexToCheck, &htc);
if (retCode_ == HBC_ERROR_INSERTROW_DUP_ROWID) {
return HBASE_DUP_ROW_ERROR;
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/exp/ExpHbaseInterface.h
----------------------------------------------------------------------
diff --git a/core/sql/exp/ExpHbaseInterface.h b/core/sql/exp/ExpHbaseInterface.h
index daba401..0bfda35 100644
--- a/core/sql/exp/ExpHbaseInterface.h
+++ b/core/sql/exp/ExpHbaseInterface.h
@@ -311,7 +311,8 @@ class ExpHbaseInterface : public NABasicObject
NABoolean useHbaseXn,
NABoolean useRegionXn,
const int64_t timestamp,
- NABoolean asyncOperation) = 0;
+ NABoolean asyncOperation,
+ Int16 colIndexToCheck) = 0;
virtual Lng32 checkAndUpdateRow(
@@ -624,7 +625,8 @@ virtual Lng32 initHFileParams(HbaseStr &tblName,
NABoolean useHbaseXn,
NABoolean useRegionXn,
const int64_t timestamp,
- NABoolean asyncOperation);
+ NABoolean asyncOperation,
+ Int16 colIndexToCheck);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/generator/GenRelUpdate.cpp
----------------------------------------------------------------------
diff --git a/core/sql/generator/GenRelUpdate.cpp b/core/sql/generator/GenRelUpdate.cpp
index 2c93d69..422e532 100644
--- a/core/sql/generator/GenRelUpdate.cpp
+++ b/core/sql/generator/GenRelUpdate.cpp
@@ -2379,6 +2379,7 @@ short HbaseInsert::codeGen(Generator *generator)
NABoolean isAlignedFormat = getTableDesc()->getNATable()->isAlignedFormat(getIndexDesc());
NABoolean isHbaseMapFormat = getTableDesc()->getNATable()->isHbaseMapTable();
+ Int16 colIndexOfPK1 = -1;
for (CollIndex ii = 0; ii < newRecExprArray().entries(); ii++)
{
@@ -2404,6 +2405,12 @@ short HbaseInsert::codeGen(Generator *generator)
space->AllocateAndCopyToAlignedSpace(cnInList, 0);
listOfOmittedColNames->insert(colNameInList);
}
+ else
+ {
+ if (col->isClusteringKey() && !isAlignedFormat && colIndexOfPK1 == -1)
+ colIndexOfPK1 = (listOfOmittedColNames == NULL) ? ii :
+ ii - listOfOmittedColNames->entries();
+ }
colArray.insert( col );
if (returnRow)
@@ -3007,6 +3014,9 @@ short HbaseInsert::codeGen(Generator *generator)
if (getTableDesc()->getNATable()->isEnabledForDDLQI())
generator->objectUids().insert(
getTableDesc()->getNATable()->objectUid().get_value());
+
+ if (colIndexOfPK1 >=0 && t == ComTdbHbaseAccess::INSERT_)
+ hbasescan_tdb->setColIndexOfPK1(colIndexOfPK1);
}
else
{
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
index 102f03b..a00e1c8 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java
@@ -1923,11 +1923,12 @@ public class HBaseClient {
Object row,
long timestamp,
boolean checkAndPut,
+ short colIndexToCheck,
boolean asyncOperation,
boolean useRegionXn) throws IOException, InterruptedException, ExecutionException {
HTableClient htc = getHTableClient(jniObject, tblName, useTRex);
- boolean ret = htc.putRow(transID, rowID, row, null, null,
+ boolean ret = htc.putRow(transID, rowID, row, null, null, colIndexToCheck,
checkAndPut, asyncOperation, useRegionXn);
if (asyncOperation == true)
htc.setJavaObject(jniObject);
@@ -1943,8 +1944,10 @@ public class HBaseClient {
boolean asyncOperation,
boolean useRegionXn) throws IOException, InterruptedException, ExecutionException {
boolean checkAndPut = true;
+ short colIndexToCheck = 0; // is overridden by columnToCheck
HTableClient htc = getHTableClient(jniObject, tblName, useTRex);
- boolean ret = htc.putRow(transID, rowID, columnsToUpdate, columnToCheck, columnValToCheck,
+ boolean ret = htc.putRow(transID, rowID, columnsToUpdate,
+ columnToCheck, columnValToCheck, colIndexToCheck,
checkAndPut, asyncOperation, useRegionXn);
if (asyncOperation == true)
htc.setJavaObject(jniObject);
http://git-wip-us.apache.org/repos/asf/incubator-trafodion/blob/921c0a58/core/sql/src/main/java/org/trafodion/sql/HTableClient.java
----------------------------------------------------------------------
diff --git a/core/sql/src/main/java/org/trafodion/sql/HTableClient.java b/core/sql/src/main/java/org/trafodion/sql/HTableClient.java
index 48cc1fe..12d134c 100644
--- a/core/sql/src/main/java/org/trafodion/sql/HTableClient.java
+++ b/core/sql/src/main/java/org/trafodion/sql/HTableClient.java
@@ -1509,7 +1509,8 @@ public class HTableClient {
public boolean putRow(final long transID, final byte[] rowID, Object row,
byte[] columnToCheck, final byte[] colValToCheck,
- final boolean checkAndPut, boolean asyncOperation,
+ final short colIndexToCheck,
+ final boolean checkAndPut, boolean asyncOperation,
final boolean useRegionXn) throws IOException, InterruptedException,
ExecutionException
{
@@ -1539,7 +1540,7 @@ public class HTableClient {
colValue = new byte[colValueLen];
bb.get(colValue, 0, colValueLen);
put.add(getFamily(colName), getName(colName), colValue);
- if (checkAndPut && colIndex == 0) {
+ if (checkAndPut && colIndex == colIndexToCheck) {
family = getFamily(colName);
qualifier = getName(colName);
}
@@ -1618,13 +1619,13 @@ public class HTableClient {
}
}
- public boolean insertRow(long transID, byte[] rowID,
+ /* public boolean insertRow(long transID, byte[] rowID,
Object row,
long timestamp,
boolean asyncOperation) throws IOException, InterruptedException, ExecutionException {
- return putRow(transID, rowID, row, null, null,
- false, asyncOperation, false);
- }
+ return putRow(transID, rowID, row, null, null, 0,
+ false, asyncOperation, false);
+ } */
public boolean putRows(final long transID, short rowIDLen, Object rowIDs,
Object rows,
@@ -1713,21 +1714,22 @@ public class HTableClient {
return true;
}
- public boolean checkAndInsertRow(long transID, byte[] rowID,
+ /* public boolean checkAndInsertRow(long transID, byte[] rowID,
Object row,
long timestamp,
boolean asyncOperation) throws IOException, InterruptedException, ExecutionException {
- return putRow(transID, rowID, row, null, null,
+ return putRow(transID, rowID, row, null, null, 0,
true, asyncOperation, false);
- }
+ } */
public boolean checkAndUpdateRow(long transID, byte[] rowID,
Object columns, byte[] columnToCheck, byte[] colValToCheck,
long timestamp, boolean asyncOperation) throws IOException, InterruptedException,
ExecutionException, Throwable {
- return putRow(transID, rowID, columns, columnToCheck,
- colValToCheck,
- true, asyncOperation, false);
+ short colIndexToCheck = 0; // overridden by columnToCheck
+ return putRow(transID, rowID, columns, columnToCheck,
+ colValToCheck, colIndexToCheck,
+ true, asyncOperation, false);
}
public byte[] coProcAggr(long transID, int aggrType,
[2/2] incubator-trafodion git commit: Merge [TRAFODION-2775] PR-1267
Insert does not raise duplicate row error for hbase format table with
defaulted first column
Posted by su...@apache.org.
Merge [TRAFODION-2775] PR-1267 Insert does not raise duplicate row error for hbase format table with defaulted first column
Project: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/commit/c66fca25
Tree: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/tree/c66fca25
Diff: http://git-wip-us.apache.org/repos/asf/incubator-trafodion/diff/c66fca25
Branch: refs/heads/master
Commit: c66fca2502db7f75a285f93ca2a83169745b25e9
Parents: d283c95 921c0a5
Author: Suresh Subbiah <su...@apache.org>
Authored: Wed Oct 18 19:14:33 2017 +0000
Committer: Suresh Subbiah <su...@apache.org>
Committed: Wed Oct 18 19:14:33 2017 +0000
----------------------------------------------------------------------
core/sql/comexe/ComTdbHbaseAccess.cpp | 2 ++
core/sql/comexe/ComTdbHbaseAccess.h | 12 ++++++++-
core/sql/executor/ExHbaseIUD.cpp | 6 +++--
core/sql/executor/HBaseClient_JNI.cpp | 7 +++---
core/sql/executor/HBaseClient_JNI.h | 1 +
core/sql/exp/ExpHbaseInterface.cpp | 9 ++++---
core/sql/exp/ExpHbaseInterface.h | 6 +++--
core/sql/generator/GenRelUpdate.cpp | 10 ++++++++
.../java/org/trafodion/sql/HBaseClient.java | 7 ++++--
.../java/org/trafodion/sql/HTableClient.java | 26 +++++++++++---------
10 files changed, 61 insertions(+), 25 deletions(-)
----------------------------------------------------------------------