You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@omid.apache.org by yo...@apache.org on 2019/03/31 09:28:44 UTC
[incubator-omid] branch 1.0.1 updated (512904c -> 167d9af)
This is an automated email from the ASF dual-hosted git repository.
yonigo pushed a change to branch 1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-omid.git.
from 512904c [ci skip] prepare release omid-1.0.1
new 7aee43f OMID-138 - Dont clear commitCache when skiping row
new dcf0444 OMID-139 - Batch transaction cleanup when aborted
new 167d9af Add Yonatans key to KEYS file
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
KEYS | 57 ++++++++++++
.../omid/transaction/HBaseSyncPostCommitter.java | 2 +-
.../apache/omid/transaction/HBaseTransaction.java | 60 +++++++++---
.../apache/omid/transaction/TestShadowCells.java | 2 +-
.../omid/transaction/TestTransactionConflict.java | 103 +++++++++++++++++++++
.../TransactionVisibilityFilterBase.java | 2 -
6 files changed, 207 insertions(+), 19 deletions(-)
[incubator-omid] 01/03: OMID-138 - Dont clear commitCache when
skiping row
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yonigo pushed a commit to branch 1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-omid.git
commit 7aee43f0ed8f493f89308f6b91dadbac47c1bcb7
Author: Yonatan Gottesman <yo...@gmail.com>
AuthorDate: Sat Mar 30 19:49:22 2019 +0300
OMID-138 - Dont clear commitCache when skiping row
---
.../org/apache/omid/transaction/TransactionVisibilityFilterBase.java | 2 --
1 file changed, 2 deletions(-)
diff --git a/hbase-coprocessor/src/main/java/org/apache/omid/transaction/TransactionVisibilityFilterBase.java b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/TransactionVisibilityFilterBase.java
index b3a6baf..7431a96 100644
--- a/hbase-coprocessor/src/main/java/org/apache/omid/transaction/TransactionVisibilityFilterBase.java
+++ b/hbase-coprocessor/src/main/java/org/apache/omid/transaction/TransactionVisibilityFilterBase.java
@@ -179,8 +179,6 @@ public class TransactionVisibilityFilterBase extends OmidFilterBase {
@Override
public void reset() throws IOException {
- commitCache.clear();
- familyDeletionCache.clear();
if (userFilter != null) {
userFilter.reset();
}
[incubator-omid] 02/03: OMID-139 - Batch transaction cleanup when
aborted
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yonigo pushed a commit to branch 1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-omid.git
commit dcf0444cfd057e0d10f173b24962ac3237fe51d5
Author: Yonatan Gottesman <yo...@gmail.com>
AuthorDate: Sun Mar 31 12:06:04 2019 +0300
OMID-139 - Batch transaction cleanup when aborted
---
.../omid/transaction/HBaseSyncPostCommitter.java | 2 +-
.../apache/omid/transaction/HBaseTransaction.java | 60 +++++++++---
.../apache/omid/transaction/TestShadowCells.java | 2 +-
.../omid/transaction/TestTransactionConflict.java | 103 +++++++++++++++++++++
4 files changed, 150 insertions(+), 17 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
index 68dc8d3..6e2f9ed 100644
--- a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseSyncPostCommitter.java
@@ -52,7 +52,7 @@ public class HBaseSyncPostCommitter implements PostCommitActions {
private final Timer commitTableUpdateTimer;
private final Timer shadowCellsUpdateTimer;
- private static final int MAX_BATCH_SIZE=1000;
+ static final int MAX_BATCH_SIZE=1000;
private final Connection connection;
public HBaseSyncPostCommitter(MetricsRegistry metrics, CommitTable.Client commitTableClient,
diff --git a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
index 62ef936..22e4b86 100644
--- a/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
+++ b/hbase-client/src/main/java/org/apache/omid/transaction/HBaseTransaction.java
@@ -18,16 +18,23 @@
package org.apache.omid.transaction;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Table;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class HBaseTransaction extends AbstractTransaction<HBaseCellId> {
private static final Logger LOG = LoggerFactory.getLogger(HBaseTransaction.class);
-
+ static final int MAX_DELETE_BATCH_SIZE = 1000;
+
public HBaseTransaction(long transactionId, long epoch, Set<HBaseCellId> writeSet,
Set<HBaseCellId> conflictFreeWriteSet, AbstractTransactionManager tm, boolean isLowLatency) {
super(transactionId, epoch, writeSet, conflictFreeWriteSet, tm, isLowLatency);
@@ -45,28 +52,51 @@ public class HBaseTransaction extends AbstractTransaction<HBaseCellId> {
super(transactionId, readTimestamp, visibilityLevel, epoch, writeSet, conflictFreeWriteSet, tm, isLowLatency);
}
- private void deleteCell(HBaseCellId cell) {
+
+ private void flushMutations(Table table, List<Mutation> mutations) throws IOException, InterruptedException {
+ table.batch(mutations, new Object[mutations.size()]);
+ }
+
+ private void deleteCell(HBaseCellId cell, Map<Table,List<Mutation>> mutations) throws IOException, InterruptedException {
+
Delete delete = new Delete(cell.getRow());
delete.addColumn(cell.getFamily(), cell.getQualifier(), cell.getTimestamp());
- try {
- cell.getTable().getHTable().delete(delete);
- } catch (IOException e) {
- LOG.warn("Failed cleanup cell {} for Tx {}. This issue has been ignored", cell, getTransactionId(), e);
+
+ Table table = cell.getTable().getHTable();
+ List<Mutation> tableMutations = mutations.get(table);
+ if (tableMutations == null) {
+ ArrayList<Mutation> newList = new ArrayList<>();
+ newList.add(delete);
+ mutations.put(table, newList);
+ } else {
+ tableMutations.add(delete);
+ if (tableMutations.size() > MAX_DELETE_BATCH_SIZE) {
+ flushMutations(table, tableMutations);
+ mutations.remove(table);
+ }
}
}
+
@Override
public void cleanup() {
- for (final HBaseCellId cell : getWriteSet()) {
- deleteCell(cell);
- }
- for (final HBaseCellId cell : getConflictFreeWriteSet()) {
- deleteCell(cell);
- }
+ Map<Table,List<Mutation>> mutations = new HashMap<>();
+
try {
- flushTables();
- } catch (IOException e) {
- LOG.warn("Failed flushing tables for Tx {}", getTransactionId(), e);
+ for (final HBaseCellId cell : getWriteSet()) {
+ deleteCell(cell, mutations);
+ }
+
+ for (final HBaseCellId cell : getConflictFreeWriteSet()) {
+ deleteCell(cell, mutations);
+ }
+
+ for (Map.Entry<Table,List<Mutation>> entry: mutations.entrySet()) {
+ flushMutations(entry.getKey(), entry.getValue());
+ }
+
+ } catch (InterruptedException | IOException e) {
+ LOG.warn("Failed cleanup for Tx {}. This issue has been ignored", getTransactionId(), e);
}
}
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
index 8cf51a2..e7712e3 100644
--- a/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestShadowCells.java
@@ -312,7 +312,7 @@ public class TestShadowCells extends OmidTestBase {
// Test shadow cells are created properly
Put put = new Put(row);
- for (int i = 0; i < 1002; ++i) {
+ for (int i = 0; i < HBaseSyncPostCommitter.MAX_BATCH_SIZE*2 + 2; ++i) {
put.addColumn(family, Bytes.toBytes(String.valueOf("X") + i), data1);
}
table.put(t1, put);
diff --git a/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
index a462d56..7181040 100644
--- a/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
+++ b/hbase-client/src/test/java/org/apache/omid/transaction/TestTransactionConflict.java
@@ -33,12 +33,15 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.ITestContext;
import org.testng.annotations.Test;
+import java.io.IOException;
+
@Test(groups = "sharedHBase")
public class TestTransactionConflict extends OmidTestBase {
@@ -278,6 +281,106 @@ public class TestTransactionConflict extends OmidTestBase {
}
+ private int countRows(Table table) throws IOException {
+ Scan scan = new Scan();
+ ResultScanner scanner = table.getScanner(scan);
+ Result r = scanner.next();
+ int rowCount = 0;
+ while (r != null) {
+ r = scanner.next();
+ rowCount++;
+ }
+ return rowCount;
+ }
+
+ @Test(timeOut = 60_000)
+ public void testBatchedCleanup(ITestContext context) throws Exception {
+
+ String table2 = "testBatchedCleanupTABLE2";
+ TableName table2Name = TableName.valueOf(table2);
+
+ try (Connection conn = ConnectionFactory.createConnection(hbaseConf);
+ Admin admin = conn.getAdmin()) {
+ TableName htable2 = TableName.valueOf(table2);
+
+ if (!admin.tableExists(htable2)) {
+ HTableDescriptor desc = new HTableDescriptor(table2Name);
+ HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
+ datafam.setMaxVersions(Integer.MAX_VALUE);
+ desc.addFamily(datafam);
+
+ admin.createTable(desc);
+ }
+
+ if (admin.isTableDisabled(htable2)) {
+ admin.enableTable(htable2);
+ }
+ }
+
+ TransactionManager tm = newTransactionManager(context);
+ TTable tt = new TTable(connection, TEST_TABLE);
+ TTable tt2 = new TTable(connection, table2);
+
+ Transaction t1 = tm.begin();
+ LOG.info("Transaction created " + t1);
+
+ Transaction t2 = tm.begin();
+ LOG.info("Transaction created" + t2);
+
+ byte[] row = Bytes.toBytes("test-simple");
+ byte[] fam = Bytes.toBytes(TEST_FAMILY);
+ byte[] col = Bytes.toBytes("testdata");
+ byte[] data1 = Bytes.toBytes("testWrite-1");
+ byte[] data2 = Bytes.toBytes("testWrite-2");
+
+ Put p = new Put(row);
+ p.addColumn(fam, col, data1);
+ tt.put(t1, p);
+
+ Get g = new Get(row).setMaxVersions();
+ g.addColumn(fam, col);
+ Result r = tt.getHTable().get(g);
+ assertEquals(r.size(), 1, "Unexpected size for read.");
+ assertTrue(Bytes.equals(data1, r.getValue(fam, col)),
+ "Unexpected value for read: " + Bytes.toString(r.getValue(fam, col)));
+
+ int rowcount = HBaseTransaction.MAX_DELETE_BATCH_SIZE*2 + 2;
+
+ // Add this row to cause conflict
+ Put p2 = new Put(row);
+ p2.addColumn(fam, col, data2);
+ tt.put(t2, p2);
+
+ //Add more rows to hit batch
+ for (int i = 0; i < rowcount; i++) {
+ byte[] newRow = Bytes.toBytes("test-del" + i);
+ Put put = new Put(newRow);
+ put.addColumn(fam, col, data2);
+ tt.put(t2, put);
+ tt2.put(t2, put);
+ }
+
+ // validate rows are really written
+ assertEquals(countRows(tt.getHTable()), rowcount + 1, "Unexpected size for read.");
+ assertEquals(countRows(tt2.getHTable()), rowcount, "Unexpected size for read.");
+
+ tm.commit(t1);
+
+ boolean aborted = false;
+ try {
+ tm.commit(t2);
+ fail("Transaction commited successfully");
+ } catch (RollbackException e) {
+ aborted = true;
+ }
+ assertTrue(aborted, "Transaction didn't raise exception");
+
+ // validate rows are cleaned
+ assertEquals(countRows(tt.getHTable()), 1, "Unexpected size for read.");
+ assertEquals(countRows(tt2.getHTable()), 0, "Unexpected size for read.");
+ }
+
+
@Test(timeOut = 10_000)
public void testMultipleCellChangesOnSameRow(ITestContext context) throws Exception {
TransactionManager tm = newTransactionManager(context);
[incubator-omid] 03/03: Add Yonatans key to KEYS file
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yonigo pushed a commit to branch 1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-omid.git
commit 167d9afa4db0b2c9bfdffe73c7d06810a96d608e
Author: Yonatan Gottesman <yo...@gmail.com>
AuthorDate: Sun Mar 31 12:28:07 2019 +0300
Add Yonatans key to KEYS file
---
KEYS | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 57 insertions(+)
diff --git a/KEYS b/KEYS
index 075352d..83ec8b1 100644
--- a/KEYS
+++ b/KEYS
@@ -235,3 +235,60 @@ E0lw41yUPwZqnEA4+pLXUBVD996e+/eP3ZhsP2dpFiFwzITaN4I1dHzmof6ULkX0
bg1uB9/EulwHtgVTF64wp1RJdb36VkD2sqyBFXnq
=QUrt
-----END PGP PUBLIC KEY BLOCK-----
+pub rsa4096 2019-03-31 [SC]
+ FDD6 BE4E 5BA7 F2C2 3E07 2F7E 9CB2 3477 E5DF B768
+uid [ultimate] Yonatan Gottesman <yo...@gmail.com>
+sub rsa4096 2019-03-31 [E]
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFygheoBEACnhosw3R81Wfs4hs9wd4Z0Fy83IX5+wQa7GkWVdiCABdQrGnXp
+q5Z5cr4NgBxKoj90PZKrBh68bA6BdLlMY4GfKG75VDRkJXChwtknvKs0V3NOT21w
+plLwv17B2hEJN/RaNhyiU5u6wXU+TPEZaMOqBlRqWqiXEAPPaHGb6S5+5SeBKlFf
+K32IjiZFjwGB03NLE8S/Zt0DWAnKyi3kR6v6iQrEFBHVzLP2lDOYT9lvCR98Vgcr
+KDBhSKr61H46gKgzm7WE6SdG33uhik3JOOzp4osPZPzyRjnClVunJwfqoHgprVY6
+XVV/U1YfZg4dN93P7mOMi/93+EIQAadHWe81ri68LnxwyHkyztVL2E1+sxA81SVX
+QU9R98YgoBzABv3RYpLPvqgCED2ZR3x73SdZ9DwCYkFSOxTDIXnpURFOHRI5UO9s
+XB4W7qqJ67TkRD7m7fzI+jononHeS9Zpf3bzaTuAKMDMm0QCOnJvOziYLLIWFiJh
+aI53sVAYH8pcw6uc4vazoaoOyfjB5BbCi+XptLMnaXG3RKDg1O/4IUmdwapT2zkN
+/twkPyDXUOdKOFZZXsargh8GRvzzbhY1JO9/diUjQV4TJwjOcAUNQ2PbHc55+HRe
+XDi2c3GbAxI0YntIF3j4UM5PfQfoFlejyTDvBMqgueO+wOh7x93hQddbUwARAQAB
+tCZZb25hdGFuIEdvdHRlc21hbiA8eW9uaWdvMTBAZ21haWwuY29tPokCTgQTAQgA
+OBYhBP3Wvk5bp/LCPgcvfpyyNHfl37doBQJcoIXqAhsDBQsJCAcCBhUKCQgLAgQW
+AgMBAh4BAheAAAoJEJyyNHfl37dobhgP/2vDQc5s98TBCe0fH/4WuA+g4aECawT9
+5mG/5sVUuA45vLYptXUJVNP+wXQuQOtDqaGaCnjvCEh35r1wuPd5mwediZ4RCrPX
+wrJEUEGtSk3WuK3ZBASKgkNpRURC9oj6g3FxHS2Jg5cuyVMWOlQIfg5Zgeih4q0y
+X/RkD3Ya7b3bc4PKJqCvfQtxvYu/rFWOHW1TQRZdbe30BNqR4inpUJTWss7fYf5W
+0DiQQjF3fF6ot8CMQN+Khu246JTLpzIOs3QgV7NLj55NCDF0N43RBnw5q7NA02SR
+CJIHD1oKdJtG+BTZU3kb+feT1vbDIbONiJrAlV1DFjk4kQ4DyMNaQtAslQwxkwRk
+qgLr5VRcgAAGMlI2fV3cjl2NBJOoxCBSIMpNEyDLLbh6X0pfspDzusr7wYIDcBeN
+crTSLiLataLvD2fx3jyzTy6mcr8XGeJ9aS38ymny0aovLYEss/VZm1B1KSjkJXLs
+YJ49BiYiRrT9nlKBXqVmrNmszSOgbA6GDWYR7H+FDUqpgF2e9wwFJGaUrOjvFPX2
+KUSLgibzMkQApYPnI2Ls5VgpxGviSQ9b3byS92wWDIhuKyzlO6G4fjs4AKbCZq/i
+hru33skE+k7SwYv0ZUqz1JGCcPEzV9scEXETMSGZlZtybdPW7HCeJ25DO4dmVqgh
+MaEvu0eSnrQJuQINBFygheoBEAC2LzHD7y6jV2uenHiXUHS6XthTAarfMhpeHH4i
+Q9DM6tPkJYyiW+N0J+6X3zWq1oRznLMCvmyEbEk9pfN2lAvyOgfLMhUtmjbsy+6e
+UXK+he13yiM1SZcASpCnxqM1uEHTsHkcDr3lmPpa5lUE3RofmkjrgmxA9/krNln2
+C6ASunkcNeiOTXG4VePFvtoPwFUO257gKu2+0H/eKNLttTQjGZqRMCYOJ5G0bE0n
+XNuon6P/8T3v3gWb6ZP0WrUSjsw2dNpji8CU8JZT9LFAD9ld77L2oqPNgwV20IgW
+gPQDtlspAqo/WKN5hJowqWB5RmHS7WOgc9OJIz+NxKzXA/vm/ve1h4mUAkFFbeiV
+j8C5WszJBkTRpOLfDZ5GTtj92INSs8BXss9xu1vNcZkJJsqR8pTKpWo6vChEyeV4
+qOI90EVHbXiOc+j6GQQPRf2yYM7cet1pzl4S6Z0t77hVhc9yz0bt7FfjHeVr5fum
+KXks4mWCNH1trH/i7PsJSQxnf8um81Bg8gPks96Rcx2PkqDUC9Oi01fgLti1G2cm
+rRzt/EilSxXMtYUBmd2Ypy9c6QM4bik7tdl4ogQbk1LPbl/c2dfyhHDBlEvZHxdo
+GLJG9uc+qgKJScar8I5/RiYiBZ7h5T2zj2/SJQ6o97FDpuIYG1Y3EV45N+Otwo/r
+bsDiKQARAQABiQI2BBgBCAAgFiEE/da+Tlun8sI+By9+nLI0d+Xft2gFAlygheoC
+GwwACgkQnLI0d+Xft2i/ZQ/+OBCGkIfYxAUG5DKoInyJGtCJxZfzMrT7P385RjHY
+0APDLqnGssaEXzkaG9qojt7jSn+ZTrkkXTNT346gNsYG9lr+CF1oyNcXA7MoCxxH
+nvGZzM6SZeqoc/X+kvOslF6KNbPB9KwBbpsLjQliF0eaMTXdVMSZwetOU9g3VRx7
+sv/xBzGCx8CJoyPJVeVsshobTZZpW+8amAxYJIhvh/BqM9xDbg1jn0Hw2mrLKBxR
+hSeXsOxnPzJS2uZFJj2H/sTjozbjZQ8Y6n6pY92w80F2evlZmPbahP3v5CrguoiC
+AovozNJrsb1w9XxYuNLqd+AM7rmEh3q2R8Rf9wE82uC/D9Y8u+Kz6kSgQ2cZ50HL
+KoYtZusHIBiaNlVLjiPF4lid3lv/GsLH6+pNQiWN8ek8zd3h0zcACMoPFFE6OS1X
+5KiUm4wYjaq6mdE36LFInMa9Yvy6LuAGvkE/4oKO7Cqd2xWX83lmZQ3uyaINuMWQ
+F8ipeWcRWgb49HmAi8+ijm5A8roEnCG8GLrE9Fp5/Q5jFG0OjWPaUovy/2LCGoSL
+SI/RuatlZpSwfXb+Isp9jHwjGt6vT9Kn6hLVcBTl3IslGTcTwAcSge/pfGcJXRqo
+INIspfCsz7SUr2B/QYDfV3D6P1CCNpK68f2mgZYMMPJiyuyXyyPcUcfArhnHF/fB
+H/k=
+=ClsZ
+-----END PGP PUBLIC KEY BLOCK-----